Nim/lib/std/widestrs.nim

#
#
#            Nim's Runtime Library
#        (c) Copyright 2012 Andreas Rumpf
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

## Nim support for C/C++'s `wide strings`:idx:.

#when not declared(ThisIsSystem):
#  {.error: "You must not import this module explicitly".}

type
  Utf16Char* = distinct int16

when not (defined(cpu16) or defined(cpu8)):
  when defined(nimv2):

    type
      WideCString* = ptr UncheckedArray[Utf16Char]

      WideCStringObj* = object
        bytes: int
        data: WideCString

    const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) or defined(gcYrc)
    when defined(nimAllowNonVarDestructor) and arcLike:
      proc `=destroy`(a: WideCStringObj) =
        if a.data != nil:
          when compileOption("threads"):
            deallocShared(a.data)
          else:
            dealloc(a.data)
    else:
      proc `=destroy`(a: var WideCStringObj) =
        if a.data != nil:
          when compileOption("threads"):
            deallocShared(a.data)
          else:
            dealloc(a.data)

    proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.}

    proc `=sink`(a: var WideCStringObj; b: WideCStringObj) =
      a.bytes = b.bytes
      a.data = b.data

    proc createWide(a: var WideCStringObj; bytes: int) =
      a.bytes = bytes
      when compileOption("threads"):
        a.data = cast[typeof(a.data)](allocShared0(bytes))
      else:
        a.data = cast[typeof(a.data)](alloc0(bytes))

    template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx]
    template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val

    template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil)

    converter toWideCString*(x: WideCStringObj): WideCString {.inline.} =
      result = x.data

  else:
    template nullWide(): untyped = nil

    type
      WideCString* = ref UncheckedArray[Utf16Char]
      WideCStringObj* = WideCString

    template createWide(a; L) =
      unsafeNew(a, L)

  proc ord(arg: Utf16Char): int = int(cast[uint16](arg))

  proc len*(w: WideCString): int =
    ## returns the length of a widestring. This traverses the whole string to
    ## find the binary zero end marker!
    result = 0
    while int16(w[result]) != 0'i16: inc result

  const
    UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16)
    UNI_MAX_BMP = 0x0000FFFF
    UNI_MAX_UTF16 = 0x0010FFFF
    # UNI_MAX_UTF32 = 0x7FFFFFFF
    # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF

    halfShift = 10
    halfBase = 0x0010000
    halfMask = 0x3FF

    UNI_SUR_HIGH_START = 0xD800
    UNI_SUR_HIGH_END = 0xDBFF
    UNI_SUR_LOW_START = 0xDC00
    UNI_SUR_LOW_END = 0xDFFF
    UNI_REPL = 0xFFFD

  template ones(n: untyped): untyped = ((1 shl n)-1)

  template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) =
    ## Returns the unicode character `s[i]` in `result`. If `doInc == true`
    ## `i` is incremented by the number of bytes that have been processed.
    bind ones

    if ord(s[i]) <= 127:
      result = ord(s[i])
      when doInc: inc(i)
    elif ord(s[i]) shr 5 == 0b110:
      #assert(ord(s[i+1]) shr 6 == 0b10)
      if i <= L - 2:
        result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6))
        when doInc: inc(i, 2)
      else:
        result = UNI_REPL
        when doInc: inc(i)
    elif ord(s[i]) shr 4 == 0b1110:
      if i <= L - 3:
        #assert(ord(s[i+1]) shr 6 == 0b10)
        #assert(ord(s[i+2]) shr 6 == 0b10)
        result = (ord(s[i]) and ones(4)) shl 12 or
                (ord(s[i+1]) and ones(6)) shl 6 or
                (ord(s[i+2]) and ones(6))
        when doInc: inc(i, 3)
      else:
        result = UNI_REPL
        when doInc: inc(i)
    elif ord(s[i]) shr 3 == 0b11110:
      if i <= L - 4:
        #assert(ord(s[i+1]) shr 6 == 0b10)
        #assert(ord(s[i+2]) shr 6 == 0b10)
        #assert(ord(s[i+3]) shr 6 == 0b10)
        result = (ord(s[i]) and ones(3)) shl 18 or
                (ord(s[i+1]) and ones(6)) shl 12 or
                (ord(s[i+2]) and ones(6)) shl 6 or
                (ord(s[i+3]) and ones(6))
        when doInc: inc(i, 4)
      else:
        result = UNI_REPL
        when doInc: inc(i)
    else:
      result = 0xFFFD
      when doInc: inc(i)

  iterator runes(s: cstring, L: int): int =
    var
      i = 0
      result: int
    while i < L:
      fastRuneAt(s, i, L, result, true)
      yield result

  proc newWideCString*(size: int): WideCStringObj =
    result = default(WideCStringObj)
    createWide(result, size * 2 + 2)

  proc newWideCString*(source: cstring, L: int): WideCStringObj =
    ## Warning:: `source` needs to be preallocated with the length `L`
    result = default(WideCStringObj)
    createWide(result, L * 2 + 2)
    var d = 0
    for ch in runes(source, L):

      if ch <= UNI_MAX_BMP:
        if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END:
          result[d] = UNI_REPLACEMENT_CHAR
        else:
          result[d] = cast[Utf16Char](uint16(ch))
      elif ch > UNI_MAX_UTF16:
        result[d] = UNI_REPLACEMENT_CHAR
      else:
        let ch = ch - halfBase
        result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START))
        inc d
        result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START))
      inc d
    result[d] = Utf16Char(0)

  proc newWideCString*(s: cstring): WideCStringObj =
    if s.isNil: return nullWide

    result = newWideCString(s, s.len)

  proc newWideCString*(s: string): WideCStringObj =
    result = newWideCString(cstring s, s.len)

  iterator decodeUtf16(w: WideCString; replacement: int): int =
    ## Looks for a terminating NUL for length
    var i = 0
    while w[i].int16 != 0'i16:
      var ch = ord(w[i])
      inc i
      if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
        # If the 16 bits following the high surrogate are NOT in the source...
        if w[i].int16 == 0'i16:
          ch = replacement #invalid UTF-16
        else:
          let ch2 = ord(w[i])
          # If it's a low surrogate, convert to UTF32:
          if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
            ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
            inc i
          else:
            ch = replacement #invalid UTF-16
      elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
        ch = replacement #invalid UTF-16
      yield ch

  iterator decodeUtf16(w: openArray[Utf16Char]; replacement: int): int =
    ## Doesn't look for terminating NUL for length, trusts `w.len`
    var i = 0
    while i < w.len:
      var ch = ord(w[i])
      inc i
      if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
        # If the 16 bits following the high surrogate are NOT in the source...
        if i >= w.len:
          ch = replacement #invalid UTF-16
        else:
          let ch2 = ord(w[i])
          # If it's a low surrogate, convert to UTF32:
          if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
            ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
            inc i
          else:
            ch = replacement #invalid UTF-16
      elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
        ch = replacement #invalid UTF-16
      yield ch

  proc addUtf8(dest: var string; rune: int) =
    if rune < 0x80:
      dest.add chr(rune)
    elif rune < 0x800:
      dest.add chr((rune shr 6) or 0xc0)
      dest.add chr((rune and 0x3f) or 0x80)
    elif rune < 0x10000:
      dest.add chr((rune shr 12) or 0xe0)
      dest.add chr(((rune shr 6) and 0x3f) or 0x80)
      dest.add chr((rune and 0x3f) or 0x80)
    elif rune <= 0x10FFFF:
      dest.add chr((rune shr 18) or 0xf0)
      dest.add chr(((rune shr 12) and 0x3f) or 0x80)
      dest.add chr(((rune shr 6) and 0x3f) or 0x80)
      dest.add chr((rune and 0x3f) or 0x80)
    else:
      # replacement char (in case user give very large number):
      dest.add chr(0xFFFD shr 12 or 0b1110_0000)
      dest.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00)
      dest.add chr(0xFFFD and ones(6) or 0b10_0000_00)


  proc `$`*(w: openArray[Utf16Char]; replacement: int = 0xFFFD): string =
    ## Decodes a length-delimited UTF-16 slice to UTF-8.
    ##
    ## Unlike the `WideCString` overloads, this preserves the provided length
    ## and does not search for a terminating NUL.
    if w.len == 0:
      result = ""
    else:
      result = newStringOfCap(w.len + w.len shr 2)
      for rune in w.decodeUtf16(replacement):
        result.addUtf8(rune)

  proc `$`*(w: WideCString; estimate: int; replacement: int = 0xFFFD): string =
    result = newStringOfCap(estimate + estimate shr 2)
    for rune in w.decodeUtf16(replacement):
      result.addUtf8(rune)

  proc `$`*(s: WideCString): string =
    result = s $ 80

  when defined(nimv2):
    proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string =
      `$`(s.data, estimate, replacement)

    proc `$`*(s: WideCStringObj): string =
      $(s.data)

    proc len*(w: WideCStringObj): int {.inline.} =
      len(w.data)