From a12c9cfb9d2e5df881ad90f0f4d7c211b8a14ce6 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Fri, 24 Jul 2015 14:38:20 -0400
Subject: [PATCH 01/22] Add MemFile text interface factored so users can
 optionally skip Nim string construction (e.g. to filter many lines out before
 building strings).  Only Windows and/or Unix line endings are supported by
 default, though fiddling with delim/eat arguments can probably support
 archaic MacOS9 files.

---
 lib/pure/memfiles.nim | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 76ff6a8e19..ab42e90c7f 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -245,3 +245,35 @@ proc close*(f: var MemFile) =
   
   if error: raiseOSError(lastErr)
 
+type Record* {.unchecked.} = object
+  Beg*: cstring
+  Len*: int
+
+iterator records*(mfile: MemFile, delim='\l', eat='\r'): Record {.inline.} =
+  proc c_memchr(cstr: cstring, c: char, n: csize): cstring {.
+       importc: "memchr", header: "<string.h>" .}
+  proc `-!`(p, q: cstring): int {.inline.} = return cast[int](p) -% cast[int](q)
+  var rec: Record
+  var End: cstring
+  rec.Beg = cast[cstring](mfile.mem)
+  var remaining = mfile.size
+  while remaining > 0:
+    End = c_memchr(rec.Beg, delim, remaining)
+    if End == nil:                                  # unterminated final record
+      rec.Len = remaining
+      yield rec
+      break
+    rec.Len = End -! rec.Beg                        # delimiter is not included
+    if eat != '\0' and rec.Len > 0 and rec.Beg[rec.Len - 1] == eat:
+      dec(rec.Len)                                  # exclude extra pre-delim ch
+    yield rec
+    rec.Beg = cast[cstring](cast[int](End) +% 1)    # skip delimiter
+    remaining = mfile.size - (rec.Beg -! cast[cstring](mfile.mem))
+
+proc toString*(rec: Record): string {.inline.} =
+  proc toNimStr(str: cstring, len: int): string {. importc: "toNimStr" .}
+  result = toNimStr(cast[cstring](rec.Beg), rec.Len)
+  result[result.len] = '\0'
+
+iterator lines*(mfile: MemFile): string {.inline.} =
+  for rec in records(mfile): yield toString(rec)

From 3f4255dd14e197a398a63c0473fcbbc8b50f419c Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Fri, 24 Jul 2015 17:23:11 -0400
Subject: [PATCH 02/22] Beg->data, Len->size to address Varriount's issue.  I
 was unaware of lower case members as a strict style convention.

---
 lib/pure/memfiles.nim | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index ab42e90c7f..8e616b1e77 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -246,8 +246,8 @@ proc close*(f: var MemFile) =
   if error: raiseOSError(lastErr)
 
 type Record* {.unchecked.} = object
-  Beg*: cstring
-  Len*: int
+  data*: cstring
+  size*: int
 
 iterator records*(mfile: MemFile, delim='\l', eat='\r'): Record {.inline.} =
   proc c_memchr(cstr: cstring, c: char, n: csize): cstring {.
@@ -255,24 +255,24 @@ iterator records*(mfile: MemFile, delim='\l', eat='\r'): Record {.inline.} =
   proc `-!`(p, q: cstring): int {.inline.} = return cast[int](p) -% cast[int](q)
   var rec: Record
   var End: cstring
-  rec.Beg = cast[cstring](mfile.mem)
+  rec.data = cast[cstring](mfile.mem)
   var remaining = mfile.size
   while remaining > 0:
-    End = c_memchr(rec.Beg, delim, remaining)
+    End = c_memchr(rec.data, delim, remaining)
     if End == nil:                                  # unterminated final record
-      rec.Len = remaining
+      rec.size = remaining
       yield rec
       break
-    rec.Len = End -! rec.Beg                        # delimiter is not included
-    if eat != '\0' and rec.Len > 0 and rec.Beg[rec.Len - 1] == eat:
-      dec(rec.Len)                                  # exclude extra pre-delim ch
+    rec.size = End -! rec.data                      # delimiter is not included
+    if eat != '\0' and rec.size > 0 and rec.data[rec.size - 1] == eat:
+      dec(rec.size)                                 # exclude extra pre-delim ch
     yield rec
-    rec.Beg = cast[cstring](cast[int](End) +% 1)    # skip delimiter
-    remaining = mfile.size - (rec.Beg -! cast[cstring](mfile.mem))
+    rec.data = cast[cstring](cast[int](End) +% 1)   # skip delimiter
+    remaining = mfile.size - (rec.data -! cast[cstring](mfile.mem))
 
 proc toString*(rec: Record): string {.inline.} =
   proc toNimStr(str: cstring, len: int): string {. importc: "toNimStr" .}
-  result = toNimStr(cast[cstring](rec.Beg), rec.Len)
+  result = toNimStr(cast[cstring](rec.data), rec.size)
   result[result.len] = '\0'
 
 iterator lines*(mfile: MemFile): string {.inline.} =

From ae303cc1c3feb7120202e7c711eb6044a3486834 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 26 Jul 2015 07:52:30 -0400
Subject: [PATCH 03/22] Rename [rR]ecord->[mM]emSlice, rec->ms, End->ending.
 Also update some comments.

---
 lib/pure/memfiles.nim | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 8e616b1e77..4f92b0e2f5 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -245,35 +245,35 @@ proc close*(f: var MemFile) =
   
   if error: raiseOSError(lastErr)
 
-type Record* {.unchecked.} = object
+type MemSlice* {.unchecked.} = object
   data*: cstring
   size*: int
 
-iterator records*(mfile: MemFile, delim='\l', eat='\r'): Record {.inline.} =
+iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
   proc c_memchr(cstr: cstring, c: char, n: csize): cstring {.
        importc: "memchr", header: "<string.h>" .}
   proc `-!`(p, q: cstring): int {.inline.} = return cast[int](p) -% cast[int](q)
-  var rec: Record
-  var End: cstring
-  rec.data = cast[cstring](mfile.mem)
+  var ms: MemSlice
+  var ending: cstring
+  ms.data = cast[cstring](mfile.mem)
   var remaining = mfile.size
   while remaining > 0:
-    End = c_memchr(rec.data, delim, remaining)
-    if End == nil:                                  # unterminated final record
-      rec.size = remaining
-      yield rec
+    ending = c_memchr(ms.data, delim, remaining)
+    if ending == nil:                               # unterminated final slice
+      ms.size = remaining                           # Weird case..check eat?
+      yield ms
       break
-    rec.size = End -! rec.data                      # delimiter is not included
-    if eat != '\0' and rec.size > 0 and rec.data[rec.size - 1] == eat:
-      dec(rec.size)                                 # exclude extra pre-delim ch
-    yield rec
-    rec.data = cast[cstring](cast[int](End) +% 1)   # skip delimiter
-    remaining = mfile.size - (rec.data -! cast[cstring](mfile.mem))
+    ms.size = ending -! ms.data                     # delim is NOT included
+    if eat != '\0' and ms.size > 0 and ms.data[ms.size - 1] == eat:
+      dec(ms.size)                                  # trim pre-delim char
+    yield ms
+    ms.data = cast[cstring](cast[int](ending) +% 1)     # skip delim
+    remaining = mfile.size - (ms.data -! cast[cstring](mfile.mem))
 
-proc toString*(rec: Record): string {.inline.} =
+proc toString*(ms: MemSlice): string {.inline.} =
   proc toNimStr(str: cstring, len: int): string {. importc: "toNimStr" .}
-  result = toNimStr(cast[cstring](rec.data), rec.size)
-  result[result.len] = '\0'
+  result = toNimStr(cast[cstring](ms.data), ms.size)
+  result[result.len] = '\0' # toNimStr copies 1 extra byte but does not NUL-term
 
 iterator lines*(mfile: MemFile): string {.inline.} =
-  for rec in records(mfile): yield toString(rec)
+  for ms in memSlices(mfile): yield toString(ms)

From 18df5dad113c24696c3cae45ffdfe663f7d7b4ef Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 26 Jul 2015 09:02:43 -0400
Subject: [PATCH 04/22] Do not read past end of input.  Ensure NUL termination.

---
 lib/system/sysstr.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim
index 326c601bda..e4673b169e 100644
--- a/lib/system/sysstr.nim
+++ b/lib/system/sysstr.nim
@@ -82,7 +82,8 @@ proc copyStr(s: NimString, start: int): NimString {.compilerProc.} =
 proc toNimStr(str: cstring, len: int): NimString {.compilerProc.} =
   result = rawNewStringNoInit(len)
   result.len = len
-  c_memcpy(result.data, str, len + 1)
+  c_memcpy(result.data, str, len)
+  result[result.len] = '\0'
 
 proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} =
   result = toNimStr(str, c_strlen(str))

From a2b6a630feaf7e6165feeb44281a8f9526f5b2a3 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 26 Jul 2015 09:17:04 -0400
Subject: [PATCH 05/22] Oops..[]= not defined yet here. Use .data instead.

---
 lib/system/sysstr.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim
index e4673b169e..9c98f84166 100644
--- a/lib/system/sysstr.nim
+++ b/lib/system/sysstr.nim
@@ -83,7 +83,7 @@ proc toNimStr(str: cstring, len: int): NimString {.compilerProc.} =
   result = rawNewStringNoInit(len)
   result.len = len
   c_memcpy(result.data, str, len)
-  result[result.len] = '\0'
+  result.data[len] = '\0'
 
 proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} =
   result = toNimStr(str, c_strlen(str))

From f2c5676767fd9d9d25fc671a5be4577d029a8a16 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Tue, 28 Jul 2015 06:45:07 -0400
Subject: [PATCH 06/22] Remove NUL termination unneeded post toNimStr fix.

---
 lib/pure/memfiles.nim | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 4f92b0e2f5..90797ecc62 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -273,7 +273,6 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
 proc toString*(ms: MemSlice): string {.inline.} =
   proc toNimStr(str: cstring, len: int): string {. importc: "toNimStr" .}
   result = toNimStr(cast[cstring](ms.data), ms.size)
-  result[result.len] = '\0' # toNimStr copies 1 extra byte but does not NUL-term
 
 iterator lines*(mfile: MemFile): string {.inline.} =
   for ms in memSlices(mfile): yield toString(ms)

From 0b4c816cba8f11c28e983818e89ee8cbf20dff9e Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Wed, 29 Jul 2015 06:14:44 -0400
Subject: [PATCH 07/22] cstring->pointer in MemSlice to make consistent with
 MemFile and also to make more clear the care required to use MemSlice
 instances. (E.g., memXXX functions rather than strXXX functions).

---
 lib/pure/memfiles.nim | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 90797ecc62..ccd45ab2ba 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -246,16 +246,16 @@ proc close*(f: var MemFile) =
   if error: raiseOSError(lastErr)
 
 type MemSlice* {.unchecked.} = object
-  data*: cstring
+  data*: pointer
   size*: int
 
 iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
-  proc c_memchr(cstr: cstring, c: char, n: csize): cstring {.
+  proc c_memchr(cstr: pointer, c: char, n: csize): pointer {.
        importc: "memchr", header: "<string.h>" .}
-  proc `-!`(p, q: cstring): int {.inline.} = return cast[int](p) -% cast[int](q)
+  proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q)
   var ms: MemSlice
-  var ending: cstring
-  ms.data = cast[cstring](mfile.mem)
+  var ending: pointer
+  ms.data = mfile.mem
   var remaining = mfile.size
   while remaining > 0:
     ending = c_memchr(ms.data, delim, remaining)
@@ -264,11 +264,11 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
       yield ms
       break
     ms.size = ending -! ms.data                     # delim is NOT included
-    if eat != '\0' and ms.size > 0 and ms.data[ms.size - 1] == eat:
+    if eat != '\0' and ms.size > 0 and cast[cstring](ms.data)[ms.size - 1] == eat:
       dec(ms.size)                                  # trim pre-delim char
     yield ms
-    ms.data = cast[cstring](cast[int](ending) +% 1)     # skip delim
-    remaining = mfile.size - (ms.data -! cast[cstring](mfile.mem))
+    ms.data = cast[pointer](cast[int](ending) +% 1)     # skip delim
+    remaining = mfile.size - (ms.data -! mfile.mem)
 
 proc toString*(ms: MemSlice): string {.inline.} =
   proc toNimStr(str: cstring, len: int): string {. importc: "toNimStr" .}

From fc54fbfb986be7ec2ec02ac27afe273ad7a28d36 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 05:28:21 -0400
Subject: [PATCH 08/22] Nix unchecked which has no semantics for objects.

---
 lib/pure/memfiles.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index ccd45ab2ba..5113197f20 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -245,7 +245,7 @@ proc close*(f: var MemFile) =
   
   if error: raiseOSError(lastErr)
 
-type MemSlice* {.unchecked.} = object
+type MemSlice* = object
   data*: pointer
   size*: int
 

From a9d09f28685aad5b4630bb3efd1276633419638b Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 12:03:11 -0400
Subject: [PATCH 09/22] Revert toNimStr fix as per Araq request.

---
 lib/system/sysstr.nim | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim
index 9c98f84166..326c601bda 100644
--- a/lib/system/sysstr.nim
+++ b/lib/system/sysstr.nim
@@ -82,8 +82,7 @@ proc copyStr(s: NimString, start: int): NimString {.compilerProc.} =
 proc toNimStr(str: cstring, len: int): NimString {.compilerProc.} =
   result = rawNewStringNoInit(len)
   result.len = len
-  c_memcpy(result.data, str, len)
-  result.data[len] = '\0'
+  c_memcpy(result.data, str, len + 1)
 
 proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} =
   result = toNimStr(str, c_strlen(str))

From 2bcd4e9a38e7d6d532fde43f9ee777a4363d383f Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 12:03:52 -0400
Subject: [PATCH 10/22] Add top of module comment. Re-order definitions to
 type, then converter, then 3 iterators lowest- to highest-level (also fastest
 to slowest) including a new intermediate iterator lines(MemFile, buffer) that
 is more like readLine(File) in case that helps anyone port code. Add doc
 comments. Also have toString just use newString+c_memcpy instead of currently
 fragile toNimStr which Araq wants a separate PR for.

---
 lib/pure/memfiles.nim | 40 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 5113197f20..51f3662de4 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -11,6 +11,9 @@
 ##
 ## This module provides support for `memory mapped files`:idx:
 ## (Posix's `mmap`:idx:) on the different operating systems.
+##
+## It also provides some fast iterators over lines in text files
+## delimited in Unix or Windows styles (or similarly delimited records).
 
 when defined(windows):
   import winlean
@@ -249,7 +252,23 @@ type MemSlice* = object
   data*: pointer
   size*: int
 
+proc c_memcpy(a, b: pointer, n: int) {.importc: "memcpy", header: "<string.h>".}
+
+proc toString*(ms: MemSlice): string {.inline.} =
+  ## Return a Nim string built from a MemSlice.
+  var buf = newString(ms.size)
+  c_memcpy(addr(buf[0]), ms.data, ms.size)
+  buf[ms.size] = '\0'
+  result = buf
+
 iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
+  ## Iterates over [optional eat]delim-delimited slices in a MemFile.
+  ## Default delimiting is [\r]\l which parse Unix or Windows text file lines.
+  ## Pass eat='\0' to be strictly delim-delimited.
+  ## This zero copy, memchr-limited method is probably the fastest way to
+  ## iterate through lines in a file, however the returned (data,size) objects
+  ## are NOT Nim strings or even terminated C strings.  So, be careful how data
+  ## is accessed (e.g., use C mem* functions, not str* functions).
   proc c_memchr(cstr: pointer, c: char, n: csize): pointer {.
        importc: "memchr", header: "<string.h>" .}
   proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q)
@@ -270,9 +289,20 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
     ms.data = cast[pointer](cast[int](ending) +% 1)     # skip delim
     remaining = mfile.size - (ms.data -! mfile.mem)
 
-proc toString*(ms: MemSlice): string {.inline.} =
-  proc toNimStr(str: cstring, len: int): string {. importc: "toNimStr" .}
-  result = toNimStr(cast[cstring](ms.data), ms.size)
+iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} =
+  ## Replace contents of passed buffer with each new line, like readLine(File).
+  ## Default delimiting is [\r]\l which parse Unix or Windows text file lines.
+  ## Pass eat='\0' to be strictly delim-delimited.
+  for ms in memSlices(mfile, delim, eat):
+    buf.setLen(ms.size)
+    c_memcpy(addr(buf[0]), ms.data, ms.size)
+    buf[ms.size] = '\0'
+    yield buf
 
-iterator lines*(mfile: MemFile): string {.inline.} =
-  for ms in memSlices(mfile): yield toString(ms)
+iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
+  ## Return each line in a file as a Nim string, like lines(File).
+  ## Default delimiting is [\r]\l which parse Unix or Windows text file lines.
+  ## Pass eat='\0' to be strictly delim-delimited.
+  var buf = TaintedString(newStringOfCap(80))
+  for line in lines(mfile, buf, delim, eat):
+    yield buf

From 2252f80e72508b6be3f16be98e0f5815cf862343 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 12:09:53 -0400
Subject: [PATCH 11/22] Add some very simple tests for memfiles iterators.

---
 tests/misc/tmemlines.nim    | 5 +++++
 tests/misc/tmemlinesBuf.nim | 6 ++++++
 tests/misc/tmemslices.nim   | 6 ++++++
 3 files changed, 17 insertions(+)
 create mode 100644 tests/misc/tmemlines.nim
 create mode 100644 tests/misc/tmemlinesBuf.nim
 create mode 100644 tests/misc/tmemslices.nim

diff --git a/tests/misc/tmemlines.nim b/tests/misc/tmemlines.nim
new file mode 100644
index 0000000000..19821ea26b
--- /dev/null
+++ b/tests/misc/tmemlines.nim
@@ -0,0 +1,5 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+for line in lines(inp):
+  echo("#" & line & "#")
+close(inp)
diff --git a/tests/misc/tmemlinesBuf.nim b/tests/misc/tmemlinesBuf.nim
new file mode 100644
index 0000000000..12160cbf91
--- /dev/null
+++ b/tests/misc/tmemlinesBuf.nim
@@ -0,0 +1,6 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+var buffer: TaintedString = ""
+for line in lines(, buffer):
+  echo("#" & line & "#")
+close(inp)
diff --git a/tests/misc/tmemslices.nim b/tests/misc/tmemslices.nim
new file mode 100644
index 0000000000..8c658ad28a
--- /dev/null
+++ b/tests/misc/tmemslices.nim
@@ -0,0 +1,6 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+for mem in memSlices(inp):
+  if mem.size > 3:
+    echo("#" & toString(mem) & "#")
+close(inp)

From 63c4f204e8cb7d505b13a6707783e50b58b413ae Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 12:20:05 -0400
Subject: [PATCH 12/22] Add some little one-line doc comment of MemSlice.

---
 lib/pure/memfiles.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 51f3662de4..f9f4965c44 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -248,7 +248,7 @@ proc close*(f: var MemFile) =
   
   if error: raiseOSError(lastErr)
 
-type MemSlice* = object
+type MemSlice* = object  ## represent slice of a MemFile for iteration over delimited lines/records
   data*: pointer
   size*: int
 

From ad67bfcf4637532538ad09ffd34dd592b201f2d2 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 12:21:49 -0400
Subject: [PATCH 13/22] Quote the \ in doc comments.

---
 lib/pure/memfiles.nim | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index f9f4965c44..d2d2a9c8fe 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -263,8 +263,8 @@ proc toString*(ms: MemSlice): string {.inline.} =
 
 iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
   ## Iterates over [optional eat]delim-delimited slices in a MemFile.
-  ## Default delimiting is [\r]\l which parse Unix or Windows text file lines.
-  ## Pass eat='\0' to be strictly delim-delimited.
+  ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
+  ## Pass eat='\\0' to be strictly delim-delimited.
   ## This zero copy, memchr-limited method is probably the fastest way to
   ## iterate through lines in a file, however the returned (data,size) objects
   ## are NOT Nim strings or even terminated C strings.  So, be careful how data
@@ -291,8 +291,8 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
 
 iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} =
   ## Replace contents of passed buffer with each new line, like readLine(File).
-  ## Default delimiting is [\r]\l which parse Unix or Windows text file lines.
-  ## Pass eat='\0' to be strictly delim-delimited.
+  ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
+  ## Pass eat='\\0' to be strictly delim-delimited.
   for ms in memSlices(mfile, delim, eat):
     buf.setLen(ms.size)
     c_memcpy(addr(buf[0]), ms.data, ms.size)
@@ -301,7 +301,7 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
   ## Return each line in a file as a Nim string, like lines(File).
-  ## Default delimiting is [\r]\l which parse Unix or Windows text file lines.
+  ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
   ## Pass eat='\0' to be strictly delim-delimited.
   var buf = TaintedString(newStringOfCap(80))
   for line in lines(mfile, buf, delim, eat):

From 2e4e0ffd3d69e37c9c6273979359d41ae65bf48a Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 12:25:57 -0400
Subject: [PATCH 14/22] Add some example code blocks.

---
 lib/pure/memfiles.nim | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index d2d2a9c8fe..e94ccaf3d9 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -268,7 +268,12 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
   ## This zero copy, memchr-limited method is probably the fastest way to
   ## iterate through lines in a file, however the returned (data,size) objects
   ## are NOT Nim strings or even terminated C strings.  So, be careful how data
-  ## is accessed (e.g., use C mem* functions, not str* functions).
+  ## is accessed (e.g., use C mem* functions, not str* functions).  Example:
+  ## .. code-block:: nim
+  ##   var count = 0
+  ##   for slice in memSlices(memfiles.open("foo")):
+  ##     inc(count)
+  ##   echo count
   proc c_memchr(cstr: pointer, c: char, n: csize): pointer {.
        importc: "memchr", header: "<string.h>" .}
   proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q)
@@ -302,7 +307,10 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
   ## Return each line in a file as a Nim string, like lines(File).
   ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
-  ## Pass eat='\0' to be strictly delim-delimited.
+  ## Pass eat='\0' to be strictly delim-delimited.  Example:
+  ## .. code-block:: nim
+  ##   for line in lines(memfiles.open("foo")):
+  ##     echo line
   var buf = TaintedString(newStringOfCap(80))
   for line in lines(mfile, buf, delim, eat):
     yield buf

From 1bea224c4a7f05cd3b3c7eef6b2246ddac371555 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Sun, 2 Aug 2015 12:29:24 -0400
Subject: [PATCH 15/22] Try to get examples to render right as HTML.

---
 lib/pure/memfiles.nim | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index e94ccaf3d9..f1bd7d11c0 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -268,12 +268,14 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
   ## This zero copy, memchr-limited method is probably the fastest way to
   ## iterate through lines in a file, however the returned (data,size) objects
   ## are NOT Nim strings or even terminated C strings.  So, be careful how data
-  ## is accessed (e.g., use C mem* functions, not str* functions).  Example:
+  ## is accessed (e.g., use C mem* functions, not str* functions). Example:
+  ##
   ## .. code-block:: nim
   ##   var count = 0
   ##   for slice in memSlices(memfiles.open("foo")):
   ##     inc(count)
   ##   echo count
+
   proc c_memchr(cstr: pointer, c: char, n: csize): pointer {.
        importc: "memchr", header: "<string.h>" .}
   proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q)
@@ -308,9 +310,11 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.}
   ## Return each line in a file as a Nim string, like lines(File).
   ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
   ## Pass eat='\0' to be strictly delim-delimited.  Example:
+  ##
   ## .. code-block:: nim
   ##   for line in lines(memfiles.open("foo")):
   ##     echo line
+
   var buf = TaintedString(newStringOfCap(80))
   for line in lines(mfile, buf, delim, eat):
     yield buf

From c6d609f88bf45371f99a1ed924dd4c4dc46ba181 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Mon, 3 Aug 2015 12:25:30 -0400
Subject: [PATCH 16/22] Fix accidentally deleted inp argument.

---
 tests/misc/tmemlinesBuf.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/misc/tmemlinesBuf.nim b/tests/misc/tmemlinesBuf.nim
index 12160cbf91..21edc23223 100644
--- a/tests/misc/tmemlinesBuf.nim
+++ b/tests/misc/tmemlinesBuf.nim
@@ -1,6 +1,6 @@
 import memfiles
 var inp = memfiles.open("readme.txt")
 var buffer: TaintedString = ""
-for line in lines(, buffer):
+for line in lines(inp, buffer):
   echo("#" & line & "#")
 close(inp)

From c373c7209ab01ce9b2570fb0134ad52457f9ee91 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Mon, 3 Aug 2015 12:26:08 -0400
Subject: [PATCH 17/22] toString --> `$`.

---
 lib/pure/memfiles.nim     | 2 +-
 tests/misc/tmemslices.nim | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index f1bd7d11c0..c646c31a18 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -254,7 +254,7 @@ type MemSlice* = object  ## represent slice of a MemFile for iteration over deli
 
 proc c_memcpy(a, b: pointer, n: int) {.importc: "memcpy", header: "<string.h>".}
 
-proc toString*(ms: MemSlice): string {.inline.} =
+proc `$`*(ms: MemSlice): string {.inline.} =
   ## Return a Nim string built from a MemSlice.
   var buf = newString(ms.size)
   c_memcpy(addr(buf[0]), ms.data, ms.size)
diff --git a/tests/misc/tmemslices.nim b/tests/misc/tmemslices.nim
index 8c658ad28a..951807cc4f 100644
--- a/tests/misc/tmemslices.nim
+++ b/tests/misc/tmemslices.nim
@@ -2,5 +2,5 @@ import memfiles
 var inp = memfiles.open("readme.txt")
 for mem in memSlices(inp):
   if mem.size > 3:
-    echo("#" & toString(mem) & "#")
+    echo("#" & $mem & "#")
 close(inp)

From 9e8b4475e25adbe76331e6ab799443b1e2b62f7f Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Mon, 3 Aug 2015 12:34:03 -0400
Subject: [PATCH 18/22] Spruce up doc comments as per dom96 suggestions.

---
 lib/pure/memfiles.nim | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index c646c31a18..f78cd8766d 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -263,12 +263,12 @@ proc `$`*(ms: MemSlice): string {.inline.} =
 
 iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
   ## Iterates over [optional eat]delim-delimited slices in a MemFile.
-  ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
+  ## Default delimiting is [\\r]\\l which parses Unix/Windows text file lines.
   ## Pass eat='\\0' to be strictly delim-delimited.
   ## This zero copy, memchr-limited method is probably the fastest way to
-  ## iterate through lines in a file, however the returned (data,size) objects
-  ## are NOT Nim strings or even terminated C strings.  So, be careful how data
-  ## is accessed (e.g., use C mem* functions, not str* functions). Example:
+  ## iterate through lines in a file.  The returned (data,size) objects are
+  ## NOT Nim strings or even terminated C strings.  So, be careful how data
+  ## is accessed (e.g., think C mem* functions, not str* functions).  Example:
   ##
   ## .. code-block:: nim
   ##   var count = 0
@@ -298,8 +298,13 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
 
 iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} =
   ## Replace contents of passed buffer with each new line, like readLine(File).
-  ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
-  ## Pass eat='\\0' to be strictly delim-delimited.
+  ## Default delimiting is [\\r]\\l which parses Unix/Windows text file lines.
+  ## Pass eat='\\0' to be strictly delim-delimited.  Example:
+  ##
+  ## .. code-block:: nim
+  ##   var buffer: TaintedString = ""
+  ##   for line in lines(memfiles.open("foo"), buffer):
+  ##     echo line
   for ms in memSlices(mfile, delim, eat):
     buf.setLen(ms.size)
     c_memcpy(addr(buf[0]), ms.data, ms.size)
@@ -308,7 +313,7 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
   ## Return each line in a file as a Nim string, like lines(File).
-  ## Default delimiting is [\\r]\\l which parse Unix or Windows text file lines.
+  ## Default delimiting is [\\r]\\l which parses Unix/Windows text file lines.
   ## Pass eat='\0' to be strictly delim-delimited.  Example:
   ##
   ## .. code-block:: nim

From 5497c964d0aaf3fa0a9b3c33b6b6691e3cc5b985 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Mon, 3 Aug 2015 12:35:57 -0400
Subject: [PATCH 19/22] Move tests to stdlib.

---
 tests/{misc => stdlib}/tmemlines.nim    | 0
 tests/{misc => stdlib}/tmemlinesBuf.nim | 0
 tests/{misc => stdlib}/tmemslices.nim   | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/{misc => stdlib}/tmemlines.nim (100%)
 rename tests/{misc => stdlib}/tmemlinesBuf.nim (100%)
 rename tests/{misc => stdlib}/tmemslices.nim (100%)

diff --git a/tests/misc/tmemlines.nim b/tests/stdlib/tmemlines.nim
similarity index 100%
rename from tests/misc/tmemlines.nim
rename to tests/stdlib/tmemlines.nim
diff --git a/tests/misc/tmemlinesBuf.nim b/tests/stdlib/tmemlinesBuf.nim
similarity index 100%
rename from tests/misc/tmemlinesBuf.nim
rename to tests/stdlib/tmemlinesBuf.nim
diff --git a/tests/misc/tmemslices.nim b/tests/stdlib/tmemslices.nim
similarity index 100%
rename from tests/misc/tmemslices.nim
rename to tests/stdlib/tmemslices.nim

From abf15ec6ceb99bed8bf79a1a7a566596d32097bc Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Tue, 4 Aug 2015 12:11:33 -0400
Subject: [PATCH 20/22] Expand memSlices doc comment & use better example. Have
 lines() iterators docs point back to memSlices() for details.

---
 lib/pure/memfiles.nim | 41 +++++++++++++++++++++++++++--------------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index f78cd8766d..a506d2c243 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -12,8 +12,8 @@
 ## This module provides support for `memory mapped files`:idx:
 ## (Posix's `mmap`:idx:) on the different operating systems.
 ##
-## It also provides some fast iterators over lines in text files
-## delimited in Unix or Windows styles (or similarly delimited records).
+## It also provides some fast iterators over lines in text files (or
+## other "line-like", variable length, delimited records).
 
 when defined(windows):
   import winlean
@@ -262,18 +262,30 @@ proc `$`*(ms: MemSlice): string {.inline.} =
   result = buf
 
 iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
-  ## Iterates over [optional eat]delim-delimited slices in a MemFile.
-  ## Default delimiting is [\\r]\\l which parses Unix/Windows text file lines.
-  ## Pass eat='\\0' to be strictly delim-delimited.
-  ## This zero copy, memchr-limited method is probably the fastest way to
-  ## iterate through lines in a file.  The returned (data,size) objects are
-  ## NOT Nim strings or even terminated C strings.  So, be careful how data
-  ## is accessed (e.g., think C mem* functions, not str* functions).  Example:
+  ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`.
+  ##
+  ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l)
+  ## style on on a line-by-line basis.  I.e., not every line needs the same ending.
+  ## Unlike readLine(File) & lines(File), archaic MacOS9 \\r-delimited lines
+  ## are not supported as a third option for each line.  Such archaic MacOS9
+  ## files can be handled by passing delim='\\r', eat='\\0', though.
+  ##
+  ## Non-default delimiters can be passed to allow iteration over other sorts
+  ## of "line-like" variable length records.  Pass eat='\\0' to be strictly
+  ## `delim`-delimited. (Eating an optional prefix equal to '\\0' is not
+  ## supported.)
+  ##
+  ## This zero copy, memchr-limited interface is probably the fastest way to
+  ## iterate over line-like records in a file.  However, returned (data,size)
+  ## objects are not Nim strings, bounds checked Nim arrays, or even terminated
+  ## C strings.  So, care is required to access the data (e.g., think C mem*
+  ## functions, not str* functions).  Example:
   ##
   ## .. code-block:: nim
   ##   var count = 0
   ##   for slice in memSlices(memfiles.open("foo")):
-  ##     inc(count)
+  ##     if slice.size > 0 and cast[cstring](slice.data)[0] != '#':
+  ##       inc(count)
   ##   echo count
 
   proc c_memchr(cstr: pointer, c: char, n: csize): pointer {.
@@ -298,13 +310,14 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
 
 iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} =
   ## Replace contents of passed buffer with each new line, like readLine(File).
-  ## Default delimiting is [\\r]\\l which parses Unix/Windows text file lines.
-  ## Pass eat='\\0' to be strictly delim-delimited.  Example:
+  ## `delim`, `eat`, and delimiting logic is exactly as for memSlices, but Nim
+  ## strings are returned.  Example:
   ##
   ## .. code-block:: nim
   ##   var buffer: TaintedString = ""
   ##   for line in lines(memfiles.open("foo"), buffer):
   ##     echo line
+
   for ms in memSlices(mfile, delim, eat):
     buf.setLen(ms.size)
     c_memcpy(addr(buf[0]), ms.data, ms.size)
@@ -313,8 +326,8 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
   ## Return each line in a file as a Nim string, like lines(File).
-  ## Default delimiting is [\\r]\\l which parses Unix/Windows text file lines.
-  ## Pass eat='\0' to be strictly delim-delimited.  Example:
+  ## `delim`, `eat`, and delimiting logic is exactly as for memSlices, but Nim
+  ## strings are returned.  Example:
   ##
   ## .. code-block:: nim
   ##   for line in lines(memfiles.open("foo")):

From 0487ad418040b0512603ec7f640307ab34895400 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Tue, 4 Aug 2015 13:16:27 -0400
Subject: [PATCH 21/22] Use hyperlinks to refer to other procs.

---
 lib/pure/memfiles.nim | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index a506d2c243..15fa4c65e1 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -309,9 +309,10 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
     remaining = mfile.size - (ms.data -! mfile.mem)
 
 iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} =
-  ## Replace contents of passed buffer with each new line, like readLine(File).
-  ## `delim`, `eat`, and delimiting logic is exactly as for memSlices, but Nim
-  ## strings are returned.  Example:
+  ## Replace contents of passed buffer with each new line, like
+  ## `readLine(File) <system.html#readLine,File,TaintedString>`_.
+  ## `delim`, `eat`, and delimiting logic is exactly as for
+  ## `memSlices <#memSlices>`_, but Nim strings are returned.  Example:
   ##
   ## .. code-block:: nim
   ##   var buffer: TaintedString = ""
@@ -325,9 +326,10 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
     yield buf
 
 iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
-  ## Return each line in a file as a Nim string, like lines(File).
-  ## `delim`, `eat`, and delimiting logic is exactly as for memSlices, but Nim
-  ## strings are returned.  Example:
+  ## Return each line in a file as a Nim string, like
+  ## `lines(File) <system.html#lines.i,File>`_.
+  ## `delim`, `eat`, and delimiting logic is exactly as for
+  ## `memSlices <#memSlices>`_, but Nim strings are returned.  Example:
   ##
   ## .. code-block:: nim
   ##   for line in lines(memfiles.open("foo")):

From 5ec4b7946fd8c916ac0cf0d94b42837a571adc68 Mon Sep 17 00:00:00 2001
From: Charles Blake <cblake@csail.mit.edu>
Date: Tue, 4 Aug 2015 13:25:41 -0400
Subject: [PATCH 22/22] Add note about what is included.

---
 lib/pure/memfiles.nim | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index 15fa4c65e1..00929eaa24 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -270,6 +270,9 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
   ## are not supported as a third option for each line.  Such archaic MacOS9
   ## files can be handled by passing delim='\\r', eat='\\0', though.
   ##
+  ## Delimiters are not part of the returned slice.  A final, unterminated line
+  ## or record is returned just like any other.
+  ##
   ## Non-default delimiters can be passed to allow iteration over other sorts
   ## of "line-like" variable length records.  Pass eat='\\0' to be strictly
   ## `delim`-delimited. (Eating an optional prefix equal to '\\0' is not