Merge branch 'devel' of github.com:nim-lang/Nim into araq-devel

lib/nintendoswitch/switch_memory.nim

@@ -1,21 +1,36 @@
+## All of these library headers and source can be found in the github repo
+## https://github.com/switchbrew/libnx.
+
 const virtMemHeader = "<switch/kernel/virtmem.h>"
-const svcHeader = "<switch/kernel/virtmem.h>"
+const svcHeader = "<switch/kernel/svc.h>"
 const mallocHeader = "<malloc.h>"
 
+## Aligns a block of memory with request `size` to `bytes` size. For
+## example, a request of memalign(0x1000, 0x1001) == 0x2000 bytes allocated
 proc memalign*(bytes: csize, size: csize): pointer {.importc: "memalign",
     header: mallocHeader.}
 
-proc free*(address: pointer) {.importc: "free",
-    header: mallocHeader.}
+# Should be required, but not needed now because of how
+# svcUnmapMemory frees all memory
+#proc free*(address: pointer) {.importc: "free",
+#    header: mallocHeader.}
 
+## Maps a memaligned block of memory from `src_addr` to `dst_addr`. The
+## Nintendo Switch requires this call in order to make use of memory, otherwise
+## an invalid memory access occurs.
 proc svcMapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {.
     importc: "svcMapMemory", header: svcHeader.}
 
+## Unmaps (frees) all memory from both `dst_addr` and `src_addr`. **Must** be called
+## whenever svcMapMemory is used. The Switch will expect all memory to be allocated
+## before gfxExit() calls (<switch/gfx/gfx.h>)
 proc svcUnmapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {.
     importc: "svcUnmapMemory", header: svcHeader.}
 
 proc virtmemReserveMap*(size: csize): pointer {.importc: "virtmemReserveMap",
     header: virtMemHeader.}
 
-proc virtmemFreeMap*(address: pointer; size: csize) {.importc: "virtmemFreeMap",
-    header: virtMemHeader.}
+# Should be required, but not needed now because of how
+# svcUnmapMemory frees all memory
+#proc virtmemFreeMap*(address: pointer; size: csize) {.importc: "virtmemFreeMap",
+#    header: virtMemHeader.}

lib/system.nim

@@ -3916,7 +3916,7 @@ proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} =
   of '\a': s.add "\\a" # \x07
   of '\b': s.add "\\b" # \x08
   of '\t': s.add "\\t" # \x09
-  of '\n': s.add "\\n" # \x0A
+  of '\L': s.add "\\n" # \x0A
   of '\v': s.add "\\v" # \x0B
   of '\f': s.add "\\f" # \x0C
   of '\c': s.add "\\c" # \x0D

lib/system/osalloc.nim

@@ -81,41 +81,115 @@ elif defined(genode):
   include genode/alloc # osAllocPages, osTryAllocPages, osDeallocPages
 
 elif defined(nintendoswitch):
+
   import nintendoswitch/switch_memory
 
-  var
-    stack: pointer
+  type
+    PSwitchBlock = ptr NSwitchBlock
+    ## This will hold the heap pointer data in a separate
+    ## block of memory that is PageSize bytes above
+    ## the requested memory. It's the only good way
+    ## to pass around data with heap allocations
+    NSwitchBlock {.pure, inheritable.} = object
+      realSize: int
+      heap: pointer           # pointer to main heap alloc
+      heapMirror: pointer     # pointer to virtmem mapped heap
 
-  proc alignSize(size: int): int =
-    (size + 0x00000FFF) and not 0x00000FFF
+  proc alignSize(size: int): int {.inline.} =
+    ## Align a size integer to be in multiples of PageSize
+    ## The nintendo switch will not allocate memory that is not
+    ## aligned to 0x1000 bytes and will just crash.
+    (size + (PageSize - 1)) and not (PageSize - 1)
 
-  proc freeMem(p: pointer, msize: int) =
-    let size = alignSize(msize)
-    discard svcUnmapMemory(p, stack, size.uint64)
-    virtmemFreeMap(p, size.csize)
-    free(stack)
+  proc deallocate(heapMirror: pointer, heap: pointer, size: int) =
+    # Unmap the allocated memory
+    discard svcUnmapMemory(heapMirror, heap, size.uint64)
+    # These should be called (theoretically), but referencing them crashes the switch.
+    # The above call seems to free all heap memory, so these are not needed.
+    # virtmemFreeMap(nswitchBlock.heapMirror, nswitchBlock.realSize.csize)
+    # free(nswitchBlock.heap)
 
-  proc osAllocPages(msize: int): pointer {.inline.} =
-    let size = alignSize(msize)
-    stack = memalign(0x1000, size)
-    result = virtmemReserveMap(size.csize)
-    let rc = svcMapMemory(result, stack, size.uint64)
+  proc freeMem(p: pointer) =
+    # Retrieve the switch block data from the pointer we set before
+    # The data is located just sizeof(NSwitchBlock) bytes below
+    # the top of the pointer to the heap
+    let
+      nswitchDescrPos = cast[ByteAddress](p) -% sizeof(NSwitchBlock)
+      nswitchBlock = cast[PSwitchBlock](nswitchDescrPos)
+
+    deallocate(
+      nswitchBlock.heapMirror, nswitchBlock.heap, nswitchBlock.realSize
+    )
+
+  proc storeHeapData(address, heapMirror, heap: pointer, size: int) {.inline.} =
+    ## Store data in the heap for deallocation purposes later
+
+    # the position of our heap pointer data. Since we allocated PageSize extra
+    # bytes, we should have a buffer on top of the requested size of at least
+    # PageSize bytes, which is much larger than sizeof(NSwitchBlock). So we
+    # decrement the address by sizeof(NSwitchBlock) and use that address
+    # to store our pointer data
+    let nswitchBlockPos = cast[ByteAddress](address) -% sizeof(NSwitchBlock)
+
+    # We need to store this in a pointer obj (PSwitchBlock) so that the data sticks
+    # at the address we've chosen. If NSwitchBlock is used here, the data will
+    # be all 0 when we try to retrieve it later.
+    var nswitchBlock = cast[PSwitchBlock](nswitchBlockPos)
+    nswitchBlock.realSize = size
+    nswitchBlock.heap = heap
+    nswitchBlock.heapMirror = heapMirror
+
+  proc getOriginalHeapPosition(address: pointer, difference: int): pointer {.inline.} =
+    ## This function sets the heap back to the originally requested
+    ## size
+    let
+      pos = cast[int](address)
+      newPos = cast[ByteAddress](pos) +% difference
+
+    return cast[pointer](newPos)
+
+  template allocPages(size: int, outOfMemoryStmt: untyped): untyped =
+    # This is to ensure we get a block of memory the requested
+    # size, as well as space to store our structure
+    let realSize = alignSize(size + sizeof(NSwitchBlock))
+
+    let heap = memalign(PageSize, realSize)
+
+    if heap.isNil:
+      outOfMemoryStmt
+
+    let heapMirror = virtmemReserveMap(realSize.csize)
+    result = heapMirror
+
+    let rc = svcMapMemory(heapMirror, heap, realSize.uint64)
+    # Any return code not equal 0 means an error in libnx
     if rc.uint32 != 0:
-      freeMem(result, size)
+      deallocate(heapMirror, heap, realSize)
+      outOfMemoryStmt
+
+    # set result to be the original size requirement
+    result = getOriginalHeapPosition(result, realSize - size)
+
+    storeHeapData(result, heapMirror, heap, realSize)
+
+  proc osAllocPages(size: int): pointer {.inline.} =
+    allocPages(size):
       raiseOutOfMem()
 
-  proc osTryAllocPages(msize: int): pointer {.inline.} =
-    let size = alignSize(msize)
-    stack = memalign(0x1000, size)
-    result = virtmemReserveMap(size.csize)
-    let rc = svcMapMemory(result, stack, size.uint64)
-    if rc.uint32 != 0:
-      freeMem(result, size)
-      result = nil
+  proc osTryAllocPages(size: int): pointer =
+    allocPages(size):
+      return nil
 
-  proc osDeallocPages(p: pointer, size: int) {.inline.} =
+  proc osDeallocPages(p: pointer, size: int) =
+    # Note that in order for the Switch not to crash, a call to
+    # deallocHeap(runFinalizers = true, allowGcAfterwards = false)
+    # must be run before gfxExit(). The Switch requires all memory
+    # to be deallocated before the graphics application has exited.
+    #
+    # gfxExit() can be found in <switch/gfx/gfx.h> in the github
+    # repo https://github.com/switchbrew/libnx
     when reallyOsDealloc:
-      freeMem(p, size)
+      freeMem(p)
 
 elif defined(posix):
   const