Merge branch 'master' of https://github.com/odin-lang/Odin

core/encoding/pem/doc.odin

@@ -0,0 +1,7 @@
+/*
+Encodes and decodes PEM formatted data.
+
+See:
+- [[ https://www.rfc-editor.org/rfc/rfc7468.html ]]
+*/
+package pem

core/encoding/pem/pem.odin

@@ -0,0 +1,299 @@
+package pem
+
+import "base:runtime"
+import "core:bufio"
+import "core:bytes"
+import "core:crypto"
+import "core:encoding/base64"
+import "core:strings"
+
+@(private)
+BASE64_FULL_LINE_LENGTH :: 64
+@(private)
+BASE64_FULL_LINE_BYTES :: (BASE64_FULL_LINE_LENGTH / 4) * 3
+@(private)
+PREFIX_BEGIN : string : "-----BEGIN "
+@(private)
+PREFIX_END : string : "-----END "
+@(private)
+SUFFIX : string : "-----"
+@(private)
+LF :: "\n"
+@(private)
+PREEB_OVERHEAD :: len(PREFIX_BEGIN) + len(SUFFIX) + len(LF)
+@(private)
+POSTEB_OVERHEAD :: len(PREFIX_END) + len(SUFFIX)
+
+// Block is a block of PEM encoded data.
+Block :: struct {
+	label: string,
+	data: [dynamic]byte,
+}
+
+LABEL_CERTIFICATE :: "CERTIFICATE" // RFC 5280
+LABEL_X509_CRL :: "X509_CRL" // RFC 5280
+LABEL_CERTIFICATE_REQUEST :: "CERTIFICATE REQUEST" // RFC 2986
+LABEL_PKCS7 :: "PKCS7" // RFC 2315
+LABEL_CMS :: "CMS" // RFC 5652
+LABEL_PRIVATE_KEY :: "PRIVATE KEY" // RFC 5208/ RFC 5958
+LABEL_ENCRYPTED_PRIVATE_KEY :: "ENCRYPTED PRIVATE KEY" // RFC 5958
+LABEL_ATTRIBUTE_CERTIFICATE :: "ATTRIBUTE CERTIFICATE" // RFC 5755
+LABEL_PUBLIC_KEY :: "PUBLIC KEY" // RFC 5280
+
+Decode_Error :: enum {
+	None,
+	Bad_Boundary,          // Invalid boundary line.
+	Bad_Label,             // Invalid label in BEGIN/END boundary line.
+	Bad_Data,              // Invalid base64 data.
+	Label_Mismatch,        // Label in END boundary line does not match.
+	Missing_End_Boundary,  // End of data without END boundary.
+}
+
+Error :: union #shared_nil {
+	runtime.Allocator_Error,
+	Decode_Error,
+}
+
+// decode decodes the first encountered PEM block, returning the resulting
+// block, remaining data, and nil if and only if (⟺) the process was
+// successful.
+//
+// Note: No PEM blocks will result in this procedure returning all nils,
+// and is not considered an error.
+@(require_results)
+decode :: proc(data: []byte, allocator := context.allocator) -> (blk: ^Block, remaining: []byte, err: Error) {
+	line: []byte
+	remaining = data
+
+	// Search for the first `preeb`.
+	label: string
+	found := false // Label is allowed to be empty.
+	for len(remaining) > 0 {
+		line, remaining = get_line(remaining)
+
+		label, found, err = parse_eb(line, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		if found {
+			break
+		}
+	}
+	if !found {
+		return nil, nil, nil
+	}
+
+	// RFC 1421: Parse header block.
+	// RFC 7468 (lax): Skip whitespace.
+
+	// Initialize the block.
+	blk = new(Block, allocator) or_return
+	if blk.data, err = make([dynamic]byte, 0, 32, allocator); err != nil {
+		free(blk, allocator)
+		return nil, nil, err
+	}
+	if blk.label, err = strings.clone(label, allocator); err != nil {
+		block_delete(blk)
+		return nil, nil, err
+	}
+
+	// Parse the `strictbase64text`.
+	l_buf: [BASE64_FULL_LINE_BYTES]byte
+	defer crypto.zero_explicit(&l_buf, size_of(l_buf))
+	base64text_loop: for len(remaining) > 0 {
+		line, remaining = get_line(remaining)
+		l := len(line)
+		switch {
+		case l == 0:
+			block_delete(blk)
+			return nil, nil, .Bad_Data
+		case line[0] == '-':
+			// Looks like we hit the `posteb`, break.
+			break base64text_loop
+		case l > BASE64_FULL_LINE_LENGTH || l & 3 != 0:
+			// Padding is mandatory, so the line length will always
+			// be a multiple of 4.
+			block_delete(blk)
+			return nil, nil, .Bad_Data
+		}
+
+		decoded, dec_err := base64.decode_into_buf(l_buf[:], transmute(string)(line))
+		if dec_err != nil {
+			block_delete(blk)
+			return nil, nil, .Bad_Data
+		}
+
+		if _, err = append(&blk.data, ..decoded); err != nil {
+			block_delete(blk)
+			return nil, nil, err
+		}
+
+		// As `strictbase64text = *base64fullline strictbase64finl`,
+		// if we did not have a full line, we must have reached
+		// `strictbase64finl`.  Grab what should be the `posteb`
+		// and break.
+		if l < BASE64_FULL_LINE_LENGTH {
+			line, remaining = get_line(remaining)
+			break
+		}
+	}
+
+	// Validate the `posteb`.
+	post_label: string
+	post_label, found, err = parse_eb(line, false)
+	if err == nil {
+		switch {
+		case !found:
+			err = .Missing_End_Boundary
+		case label != post_label:
+			err = .Label_Mismatch
+		}
+	}
+	if err != nil {
+		block_delete(blk)
+		blk, remaining = nil, nil
+	}
+
+	return
+}
+
+// encode encodes the specified label and data into PEM format.
+@(require_results)
+encode :: proc(label: string, data: []byte, newline := false, allocator := context.allocator) -> (res: []byte, err: runtime.Allocator_Error) #optional_allocator_error {
+	sanitize_sb := proc(sb: ^strings.Builder) {
+		buf := sb.buf[:]
+		b, l := raw_data(buf), len(buf)
+		crypto.zero_explicit(b, l)
+		strings.builder_destroy(sb)
+	}
+
+	sb := strings.builder_make_none(allocator) or_return
+	defer sanitize_sb(&sb)
+
+	label_len := len(label)
+
+	// Write `preeb`.
+	n := strings.write_string(&sb, PREFIX_BEGIN)
+	n += strings.write_string(&sb, label)
+	n += strings.write_string(&sb, SUFFIX)
+	n += strings.write_string(&sb, LF)
+	if n != PREEB_OVERHEAD + label_len {
+		return nil, .Out_Of_Memory
+	}
+
+	// RFC 1421: Write header block.
+
+	// Write `base64text`.
+	l: [BASE64_FULL_LINE_LENGTH]byte
+	defer crypto.zero_explicit(&l, size_of(l))
+
+	d := data
+	for len(d) > 0 {
+		n = min(len(d), BASE64_FULL_LINE_BYTES)
+		encoded, _ := base64.encode_into_buf(l[:], d[:n])
+		d = d[n:]
+
+		expected_len := len(encoded) + len(LF)
+		n = strings.write_bytes(&sb, encoded)
+		n += strings.write_string(&sb, LF)
+		if n != expected_len {
+			return nil, .Out_Of_Memory
+		}
+	}
+
+	// Write `posteb`.
+	expected_len := POSTEB_OVERHEAD + label_len + (len(LF) if newline else 0)
+	n = strings.write_string(&sb, PREFIX_END)
+	n += strings.write_string(&sb, label)
+	n += strings.write_string(&sb, SUFFIX)
+	if newline {
+		n += strings.write_string(&sb, LF)
+	}
+	if n != expected_len {
+		return nil, .Out_Of_Memory
+	}
+
+	res = transmute([]byte)(strings.clone(strings.to_string(sb), allocator) or_return)
+
+	return
+}
+
+// block_bytes returns a slice to the Block's data.
+block_bytes :: proc(blk: ^Block) -> []byte {
+	return blk.data[:]
+}
+
+// block_delete frees a Block returned from decode.
+//
+// Note: No allocator is specified as decode uses the same allocator
+// for everything.
+block_delete :: proc(blk: ^Block) {
+	allocator := ((^runtime.Raw_Dynamic_Array)(&blk.data)).allocator
+
+	delete(blk.label, allocator)
+	sanitize_and_delete(blk.data)
+	free(blk, allocator)
+}
+
+@(private)
+get_line :: proc(data: []byte) -> (line, rest: []byte) {
+	adv: int
+	adv, line, _, _ = bufio.scan_lines(data, true)
+	rest = data[adv:]
+
+	return
+}
+
+@(private)
+parse_eb :: proc(line: []byte, is_pre: bool) -> (label: string, found: bool, err: Error) {
+	line := line
+
+	prefix: string
+	switch is_pre {
+	case true:
+		prefix = PREFIX_BEGIN
+	case false:
+		prefix = PREFIX_END
+	}
+
+	l := len(line)
+	line = bytes.trim_prefix(line, transmute([]byte)(prefix))
+	if len(line) == l {
+		return "", false, nil
+	}
+
+	l = len(line)
+	line = bytes.trim_suffix(line, transmute([]byte)(SUFFIX))
+	if len(line) == l {
+		return "", false, .Bad_Boundary
+	}
+
+	// labelchar  = %x21-2C / %x2E-7E ; any printable character,
+	//                                ; except hyphen-minus
+	// label      = [ labelchar *( ["-" / SP] labelchar ) ] ; empty ok
+	l = len(line)
+	line = bytes.trim(line, []byte{'-', ' '})
+	if len(line) != l {
+		return "", false, .Bad_Label
+	}
+	for b in line {
+		// We already ruled out non-labelchar start/end, so this
+		// allows ' '/'-'.
+		if b < 0x20 || b > 0x7e {
+			return "", false, .Bad_Label
+		}
+	}
+
+	found = true
+	label = transmute(string)(line)
+
+	return
+}
+
+@(private)
+sanitize_and_delete :: proc(data: [dynamic]byte) {
+	b, l := raw_data(data), len(data)
+	crypto.zero_explicit(b, l)
+
+	delete(data)
+}

core/slice/slice.odin

@@ -433,6 +433,9 @@ fill :: proc "contextless" (array: $T/[]$E, value: E) #no_bounds_check {
 }
 
 rotate_left :: proc "contextless" (array: $T/[]$E, mid: int) {
+	if len(a) == 0 {
+		return
+	}
 	n := len(array)
 	m := mid %% n
 	k := n - m

core/sys/info/platform_linux.odin

@@ -2,6 +2,7 @@ package sysinfo
 
 import "base:intrinsics"
 import "base:runtime"
+import "core:strconv"
 import "core:strings"
 import "core:sys/linux"
 
@@ -80,16 +81,98 @@ _os_version :: proc (allocator: runtime.Allocator, loc := #caller_location) -> (
 
 @(private)
 _ram_stats :: proc "contextless" () -> (total_ram, free_ram, total_swap, free_swap: i64, ok: bool) {
-	// Retrieve RAM info using `sysinfo`
-	sys_info: linux.Sys_Info
-	errno := linux.sysinfo(&sys_info)
-	assert_contextless(errno == .NONE, "Good luck to whoever's debugging this, something's seriously cucked up!")
+	// This is here for some of the strings procedures
+	context = runtime.default_context()
 
-	total_ram  = i64(sys_info.totalram)  * i64(sys_info.mem_unit)
-	free_ram   = i64(sys_info.freeram)   * i64(sys_info.mem_unit)
-	total_swap = i64(sys_info.totalswap) * i64(sys_info.mem_unit)
-	free_swap  = i64(sys_info.freeswap)  * i64(sys_info.mem_unit)
-	ok         = true
+	// The approach is to read /proc/meminfo for the memory information. We do this over
+	// reading sysinfo() since sysinfo() only returns MemFree, which is based on the amount
+	// of free pages. The value we actually want is MemAvailable inside meminfo since it is
+	// estimated around being about to evict things out of the page cache.
+	fd, errno := linux.open("/proc/meminfo", {})
+	if errno != .NONE {
+		// This should never happen since something would be wrong with the system
+		// if /proc/meminfo wasn't able to be opened for any reason. But, in the
+		// event that this _does_ happen, let's just try to recover through the
+		// syscall
+		sys_info: linux.Sys_Info
+		sysinfo_errno := linux.sysinfo(&sys_info)
+		assert_contextless(sysinfo_errno == .NONE, "If this has failed, there is no recovery from this")
+
+		total_ram = i64(sys_info.totalram) * i64(sys_info.mem_unit)
+		free_ram = i64(sys_info.freeram) * i64(sys_info.mem_unit)
+		total_swap = i64(sys_info.totalswap) * i64(sys_info.mem_unit)
+		free_swap = i64(sys_info.freeswap) * i64(sys_info.mem_unit)
+
+		ok = true
+
+		return
+	}
+
+	defer linux.close(fd)
+
+	// We need a relatively large size to store all the info
+	meminfo_buf: [4096]u8
+	n, read_errno := linux.read(fd, meminfo_buf[:])
+	if read_errno != .NONE {
+		sys_info: linux.Sys_Info
+		sysinfo_errno := linux.sysinfo(&sys_info)
+		assert_contextless(sysinfo_errno == .NONE, "If this has failed, there is no recovery from this")
+
+		total_ram = i64(sys_info.totalram) * i64(sys_info.mem_unit)
+		free_ram = i64(sys_info.freeram) * i64(sys_info.mem_unit)
+		total_swap = i64(sys_info.totalswap) * i64(sys_info.mem_unit)
+		free_swap = i64(sys_info.freeswap) * i64(sys_info.mem_unit)
+
+		ok = true
+
+		return
+	}
+	meminfo := string(meminfo_buf[:n])
+
+	// Fallback in the event MemAvailable is not found or is invalid in its value
+	mem_free: i64
+
+	mem_unit :: 1024
+	for line in strings.split_lines_iterator(&meminfo) {
+		if len(line) == 0 {
+			continue
+		}
+
+		colon_idx := strings.index(line, ":")
+		if colon_idx < 0 {
+			continue
+		}
+
+		key := strings.trim_space(line[:colon_idx])
+		value_str := strings.trim_space(strings.trim_suffix(line[colon_idx + 1:], "kB"))
+
+		value, conv_ok := strconv.parse_i64(value_str, 10)
+		if !conv_ok {
+			continue
+		}
+
+		switch key {
+		case "MemTotal":
+			total_ram = value * mem_unit
+		case "MemFree":
+			mem_free = value * mem_unit
+		case "MemAvailable":
+			free_ram = value * mem_unit
+		case "SwapTotal":
+			total_swap = value * mem_unit
+		case "SwapFree":
+			free_swap = value * mem_unit
+		}
+	}
+
+	if free_ram == 0 || free_ram > total_ram {
+		// We opt to return MemFree here if MemAvailable is not found or is broken to some degree.
+		// This will act as a predictable fallback, but shouldn't ever really occur unless the user
+		// is on Linux < 3.14
+		free_ram = mem_free
+	}
+
+	ok = true
 
 	return
 }

examples/all/all_js.odin

@@ -71,6 +71,7 @@ package all
 @(require) import "core:encoding/hxa"
 @(require) import "core:encoding/ini"
 @(require) import "core:encoding/json"
+@(require) import "core:encoding/pem"
 @(require) import "core:encoding/varint"
 @(require) import "core:encoding/xml"
 @(require) import "core:encoding/uuid"

examples/all/all_main.odin

@@ -76,6 +76,7 @@ package all
 @(require) import "core:encoding/hxa"
 @(require) import "core:encoding/ini"
 @(require) import "core:encoding/json"
+@(require) import "core:encoding/pem"
 @(require) import "core:encoding/varint"
 @(require) import "core:encoding/xml"
 @(require) import "core:encoding/uuid"

src/check_expr.cpp

@@ -2131,6 +2131,10 @@ gb_internal bool check_binary_op(CheckerContext *c, Operand *o, Token op) {
 		/*fallthrough*/
 	case Token_Mul:
 	case Token_MulEq:
+		if (is_type_bit_set(type)) {
+			error(op, "Operator '%.*s' is not allowed with bit sets", LIT(op.string));
+			return false;
+		}
 	case Token_AddEq:
 		if (is_type_bit_set(type)) {
 			return true;

src/llvm_backend_expr.cpp

@@ -3450,6 +3450,11 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left
 	}
 	if (is_type_array_like(a)) {
 		Type *tl = base_type(a);
+		bool inline_array_arith = lb_can_try_to_inline_array_arith(tl);
+		if (inline_array_arith && is_type_bit_field(left.type)) {
+			left = lb_emit_transmute(p, left, tl);
+			right = lb_emit_transmute(p, right, tl);
+		}
 		lbValue lhs = lb_address_from_load_or_generate_local(p, left);
 		lbValue rhs = lb_address_from_load_or_generate_local(p, right);
 
@@ -3464,7 +3469,6 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left
 			cmp_op = Token_And;
 		}
 
-		bool inline_array_arith = lb_can_try_to_inline_array_arith(tl);
 		i32 count = 0;
 		switch (tl->kind) {
 		case Type_Array:           count = cast(i32)tl->Array.count;           break;

tests/core/encoding/pem/pem.odin

@@ -0,0 +1,86 @@
+package test_core_pem
+
+import "core:bytes"
+import "core:encoding/hex"
+import "core:encoding/pem"
+import "core:testing"
+
+// RFC 7468 Section 9.
+@(private)
+CMS_PEM_TEXT : string : \
+`-----BEGIN CMS-----
+MIGDBgsqhkiG9w0BCRABCaB0MHICAQAwDQYLKoZIhvcNAQkQAwgwXgYJKoZIhvcN
+AQcBoFEET3icc87PK0nNK9ENqSxItVIoSa0o0S/ISczMs1ZIzkgsKk4tsQ0N1nUM
+dvb05OXi5XLPLEtViMwvLVLwSE0sKlFIVHAqSk3MBkkBAJv0Fx0=
+-----END CMS-----`
+@(private)
+CMS_PEM_PAYLOAD : string : "308183060b2a864886f70d0109100109a0743072020100300d060b2a864886f70d0109100308305e06092a864886f70d010701a051044f789c73cecf2b49cd2bd10da92c48b5522849ad28d12fc849ccccb35648ce482c2a4e2db10d0dd6750c76f6f4e4e5e2e572cf2c4b5588cc2f2d52f0484d2c2a514854702a4a4dcc064901009bf4171d"
+@(private)
+NOT_PEM_TEXT : string : \
+`
+Socialism is not in the least what it pretends to be.
+It is not the pioneer of a better and finer world, but the spoiler of what thousands of years of civilization have created.
+It does not build, it destroys.
+For destruction is the essence of it.
+It produces nothing, it only consumes what the social order based on private ownership in the means of production has created. `
+@(private)
+COMMENT_TEXT : string : "# 9.  Textual Encoding of Cryptographic Message Syntax"
+
+@(test)
+test_pem_roundtrip :: proc(t: ^testing.T) {
+	// Decode.
+	blk, remaining, err := pem.decode(transmute([]byte)(CMS_PEM_TEXT))
+	if !testing.expectf(t, err == nil, "PEM decode failed: %v", err) {
+		return
+	}
+	defer pem.block_delete(blk)
+
+	if !testing.expectf(t, len(remaining) == 0, "PEM decode left trailing garbage: '%s'", remaining) {
+		return
+	}
+
+	// Ensure contents match.
+	if !testing.expectf(t, blk.label == pem.LABEL_CMS, "PEM unexpected label: '%s'", blk.label) {
+		return
+	}
+	expected_payload, _ := hex.decode(transmute([]byte)(CMS_PEM_PAYLOAD))
+	defer delete(expected_payload)
+
+	if !testing.expectf(t, bytes.equal(pem.block_bytes(blk), expected_payload), "PEM unexpected data: '%x'", blk.data) {
+		return
+	}
+
+	// Encode and compare.
+	encoded := pem.encode(blk.label, pem.block_bytes(blk))
+	defer delete(encoded)
+	testing.expectf(t, CMS_PEM_TEXT == transmute(string)(encoded), "PEM encode mismatch: '%s'", encoded)
+}
+
+@(test)
+test_pem_no_blocks :: proc(t: ^testing.T) {
+	blk, remaining, err := pem.decode(transmute([]byte)(NOT_PEM_TEXT))
+	testing.expect(t, blk == nil)
+	testing.expect(t, len(remaining) == 0)
+	testing.expect(t, err == nil)
+}
+
+@(test)
+test_pem_surrounded :: proc(t: ^testing.T) {
+	blob := COMMENT_TEXT + "\n" + CMS_PEM_TEXT + "\n" + NOT_PEM_TEXT
+
+	// Should skip `COMMENT_TEXT`
+	blk, remaining, err := pem.decode(transmute([]byte)(blob))
+	if !testing.expectf(t, err == nil, "PEM decode failed: %v", err) {
+		return
+	}
+	defer pem.block_delete(blk)
+
+	// Check if the decode is correct by ensuring it round-trips.
+	encoded := pem.encode(blk.label, pem.block_bytes(blk))
+	defer delete(encoded)
+	if !testing.expectf(t, CMS_PEM_TEXT == transmute(string)(encoded), "PEM encode mismatch: '%s'", encoded) {
+		return
+	}
+
+	testing.expectf(t, NOT_PEM_TEXT == transmute(string)(remaining), "PEM remaining not preserved: '%s'", remaining)
+}