sockets.recv optimizations; stdlib now supports taint mode

2026-04-19 05:50:30 +00:00 · 2011-09-24 20:22:53 +02:00
parent 485c371942
commit 0f37d0e1f2
14 changed files with 112 additions and 74 deletions
--- a/doc/manual.txt
+++ b/doc/manual.txt
@@ -3402,3 +3402,25 @@ The interaction between threads and exceptions is simple: A *handled* exception
 in one thread cannot affect any other thread. However, an *unhandled* 
 exception in one thread terminates the whole *process*!

+Taint mode
+==========
+
+The Nimrod compiler and standard library support a `taint mode`:idx:. 
+Input strings are declared with the `TaintedString`:idx: string type declared
+in the ``system`` module.
+
+If the taint mode is turned on (via the ``--taintMode:on`` command line 
+option) it is a distinct string type which helps to detect input
+validation errors:
+
+.. code-block:: nimrod
+  echo "your name: "
+  var name: TaintedString = stdin.readline
+  # it is safe here to output the name without any input validation, so
+  # we simply convert `name` to string to make the compiler happy: 
+  echo "hi, ", name.string
+
+If the taint mode is turned off, ``TaintedString`` is simply an alias for
+``string``.
+
+
--- a/lib/impure/rdstdin.nim
+++ b/lib/impure/rdstdin.nim
@@ -14,7 +14,7 @@
 ## wanted functionality.

 when defined(Windows):
-  proc ReadLineFromStdin*(prompt: string): string = 
+  proc ReadLineFromStdin*(prompt: string): TaintedString = 
    ## Reads a line from stdin.
    stdout.write(prompt)
    result = readLine(stdin)
@@ -22,11 +22,11 @@ when defined(Windows):
 else:
  import readline, history
    
-  proc ReadLineFromStdin*(prompt: string): string = 
+  proc ReadLineFromStdin*(prompt: string): TaintedString = 
    var buffer = readline.readLine(prompt)
    if isNil(buffer): quit(0)
-    result = $buffer
-    if result.len > 0:
+    result = TaintedString($buffer)
+    if result.string.len > 0:
      add_history(buffer)
    readline.free(buffer)

--- a/lib/impure/ssl.nim
+++ b/lib/impure/ssl.nim
@@ -50,11 +50,11 @@ proc connect*(sock: var TSecureSocket, address: string,
  
  result = SSL_get_verify_result(sock.ssl)

-proc recvLine*(sock: TSecureSocket, line: var string): bool =
+proc recvLine*(sock: TSecureSocket, line: var TaintedString): bool =
  ## Acts in a similar fashion to the `recvLine` in the sockets module.
  ## Returns false when no data is available to be read.
  ## `Line` must be initialized and not nil!
-  setLen(line, 0)
+  setLen(line.string, 0)
  while True:
    var c: array[0..0, char]
    var n = BIO_read(sock.bio, c, c.len)
@@ -66,7 +66,7 @@ proc recvLine*(sock: TSecureSocket, line: var string): bool =
      elif n <= 0:
        return False
    elif c[0] == '\L': return True
-    add(line, c)
+    add(line.string, c)


 proc send*(sock: TSecureSocket, data: string) =
--- a/lib/impure/web.nim
+++ b/lib/impure/web.nim
@@ -43,19 +43,19 @@ proc URLretrieveStream*(url: string): PStream =
  if easy_perform(hCurl) != E_OK: return nil
  easy_cleanup(hCurl)
  
-proc URLretrieveString*(url: string): string = 
+proc URLretrieveString*(url: string): TaintedString = 
  ## retrieves the given `url` and returns the contents. Returns nil if an
  ## error occurs.
  var stream = newStringStream()
  var hCurl = easy_init()
-  if hCurl == nil: return nil
-  if easy_setopt(hCurl, OPT_URL, url) != E_OK: return nil
+  if hCurl == nil: return
+  if easy_setopt(hCurl, OPT_URL, url) != E_OK: return
  if easy_setopt(hCurl, OPT_WRITEFUNCTION, 
-                      curlwrapperWrite) != E_OK: return nil
-  if easy_setopt(hCurl, OPT_WRITEDATA, stream) != E_OK: return nil
-  if easy_perform(hCurl) != E_OK: return nil
+                      curlwrapperWrite) != E_OK: return
+  if easy_setopt(hCurl, OPT_WRITEDATA, stream) != E_OK: return
+  if easy_perform(hCurl) != E_OK: return
  easy_cleanup(hCurl)
-  result = stream.data
+  result = stream.data.TaintedString

 when isMainModule:
  echo URLretrieveString("http://nimrod.ethexor.com/")
--- a/lib/pure/cgi.nim
+++ b/lib/pure/cgi.nim
@@ -111,19 +111,19 @@ proc getEncodedData(allowedMethods: set[TRequestMethod]): string =
  of "POST":
    if methodPost notin allowedMethods:
      cgiError("'REQUEST_METHOD' 'POST' is not supported")
-    var L = parseInt(getenv("CONTENT_LENGTH"))
+    var L = parseInt(getenv("CONTENT_LENGTH").string)
    result = newString(L)
    if readBuffer(stdin, addr(result[0]), L) != L:
      cgiError("cannot read from stdin")
  of "GET":
    if methodGet notin allowedMethods:
      cgiError("'REQUEST_METHOD' 'GET' is not supported")
-    result = getenv("QUERY_STRING")
+    result = getenv("QUERY_STRING").string
  else:
    if methodNone notin allowedMethods:
      cgiError("'REQUEST_METHOD' must be 'POST' or 'GET'")

-iterator decodeData*(data: string): tuple[key, value: string] =
+iterator decodeData*(data: string): tuple[key, value: TaintedString] =
  ## Reads and decodes CGI data and yields the (name, value) pairs the
  ## data consists of.
  var i = 0
@@ -160,13 +160,13 @@ iterator decodeData*(data: string): tuple[key, value: string] =
      of '&', '\0': break
      else: add(value, data[i])
      inc(i)
-    yield (name, value)
+    yield (name.TaintedString, value.TaintedString)
    if data[i] == '&': inc(i)
    elif data[i] == '\0': break
    else: cgiError("'&' expected")

 iterator decodeData*(allowedMethods: set[TRequestMethod] =
-       {methodNone, methodPost, methodGet}): tuple[key, value: string] =
+       {methodNone, methodPost, methodGet}): tuple[key, value: TaintedString] =
  ## Reads and decodes CGI data and yields the (name, value) pairs the
  ## data consists of. If the client does not use a method listed in the
  ## `allowedMethods` set, an `ECgi` exception is raised.
@@ -181,7 +181,7 @@ proc readData*(allowedMethods: set[TRequestMethod] =
  ## `allowedMethods` set, an `ECgi` exception is raised.
  result = newStringTable()
  for name, value in decodeData(allowedMethods):
-    result[name] = value
+    result[name.string] = value.string

 proc validateData*(data: PStringTable, validKeys: openarray[string]) =
  ## validates data; raises `ECgi` if this fails. This checks that each variable
--- a/lib/pure/httpclient.nim
+++ b/lib/pure/httpclient.nim
@@ -75,7 +75,7 @@ proc fileError(msg: string) =
 proc charAt(d: var string, i: var int, s: TSocket): char {.inline.} = 
  result = d[i]
  while result == '\0':
-    d = s.recv()
+    d = string(s.recv())
    i = 0
    result = d[i]

@@ -98,7 +98,7 @@ proc parseChunks(d: var string, start: int, s: TSocket): string =
        digitFound = true
        chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('A') + 10)
      of '\0': 
-        d = s.recv()
+        d = string(s.recv())
        i = -1
      else: break
      inc(i)
@@ -123,7 +123,7 @@ proc parseChunks(d: var string, start: int, s: TSocket): string =
        inc(L, bytesRead)
        dec(missing, bytesRead)
      # next chunk:
-      d = s.recv()
+      d = string(s.recv())
      i = 0
    # skip trailing CR-LF:
    while charAt(d, i, s) in {'\C', '\L'}: inc(i)
@@ -139,7 +139,7 @@ proc parseBody(d: var string, start: int, s: TSocket,
    var contentLengthHeader = headers["Content-Length"]
    if contentLengthHeader != "":
      var length = contentLengthHeader.parseint()
-      while result.len() < length: result.add(s.recv())
+      while result.len() < length: result.add(s.recv.string)
    else:
      # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO
      
@@ -147,12 +147,12 @@ proc parseBody(d: var string, start: int, s: TSocket,
      # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
      if headers["Connection"] == "close":
        while True:
-          var moreData = recv(s)
+          var moreData = recv(s).string
          if moreData.len == 0: break
          result.add(moreData)

 proc parseResponse(s: TSocket): TResponse =
-  var d = s.recv()
+  var d = s.recv.string
  var i = 0

  # Parse the version
--- a/lib/pure/httpserver.nim
+++ b/lib/pure/httpserver.nim
@@ -113,11 +113,11 @@ proc executeCgi(client: TSocket, path, query: string, meth: TRequestMethod) =
    env["REQUEST_METHOD"] = "GET"
    env["QUERY_STRING"] = query
  of reqPost:
-    var buf = ""
+    var buf = TaintedString""
    var dataAvail = false
    while dataAvail:
      dataAvail = recvLine(client, buf)
-      var L = toLower(buf)
+      var L = toLower(buf.string)
      if L.startsWith("content-length:"):
        var i = len("content-length:")
        while L[i] in Whitespace: inc(i)
@@ -146,7 +146,7 @@ proc executeCgi(client: TSocket, path, query: string, meth: TRequestMethod) =
  var outp = process.outputStream
  while running(process) or not outp.atEnd(outp):
    var line = outp.readLine()
-    send(client, line)
+    send(client, line.string)
    send(client, wwwNL)

 # --------------- Server Setup -----------------------------------------------
@@ -154,10 +154,10 @@ proc executeCgi(client: TSocket, path, query: string, meth: TRequestMethod) =
 proc acceptRequest(client: TSocket) =
  var cgi = false
  var query = ""
-  var buf = ""
+  var buf = TaintedString""
  discard recvLine(client, buf)
  var path = ""
-  var data = buf.split()
+  var data = buf.string.split()
  var meth = reqGet

  var q = find(data[1], '?')
@@ -231,7 +231,7 @@ proc next*(s: var TServer) =
  ## proceed to the first/next request.
  s.client = accept(s.socket)
  headers(s.client, "")
-  var data = recv(s.client)
+  var data = recv(s.client).string
  #discard recvLine(s.client, data)
  
  var i = skipWhitespace(data)
--- a/lib/pure/irc.nim
+++ b/lib/pure/irc.nim
@@ -205,15 +205,15 @@ proc poll*(irc: var TIRC, ev: var TIRCEvent,
  ## This function should be called often as it also handles pinging
  ## the server.
  if not irc.connected: ev.typ = EvDisconnected
-  var line = ""
+  var line = TaintedString""
  var socks = @[irc.sock]
  var ret = socks.select(timeout)
  if socks.len() == 0 and ret == 1:
    if irc.sock.recvLine(line):
-      if line == "":
+      if line.string.len == 0:
        ev.typ = EvDisconnected
      else:
-        ev = parseMessage(line)
+        ev = parseMessage(line.string)
        if ev.cmd == MPing:
          irc.send("PONG " & ev.params[0])
        if ev.cmd == MPong:
--- a/lib/pure/osproc.nim
+++ b/lib/pure/osproc.nim
@@ -41,7 +41,7 @@ type

 proc execProcess*(command: string,
                  options: set[TProcessOption] = {poStdErrToStdOut,
-                                                  poUseShell}): string {.
+                                                  poUseShell}): TaintedString {.
                                                  rtl, extern: "nosp$1".}
  ## A convenience procedure that executes ``command`` with ``startProcess``
  ## and returns its output as a string.
@@ -203,13 +203,13 @@ proc select*(readfds: var seq[PProcess], timeout = 500): int
 when not defined(useNimRtl):
  proc execProcess(command: string,
                   options: set[TProcessOption] = {poStdErrToStdOut,
-                                                   poUseShell}): string =
+                                                   poUseShell}): TaintedString =
    var p = startProcessAux(command, options=options)
    var outp = outputStream(p)
-    result = ""
+    result = TaintedString""
    while running(p) or not outp.atEnd(outp):
-      result.add(outp.readLine())
-      result.add("\n")
+      result.string.add(outp.readLine().string)
+      result.string.add("\n")
    outp.close(outp)
    close(p)

--- a/lib/pure/redis.nim
+++ b/lib/pure/redis.nim
@@ -49,7 +49,7 @@ proc raiseNoOK(status: string) =
    raise newException(EInvalidReply, "Expected \"OK\" got \"$1\"" % status)

 proc parseStatus(r: TRedis): TRedisStatus =
-  var line = r.socket.recv()
+  var line = r.socket.recv.string
  
  if line[0] == '-':
    raise newException(ERedis, strip(line))
@@ -59,7 +59,7 @@ proc parseStatus(r: TRedis): TRedisStatus =
  return line.substr(1, line.len-3) # Strip '+' and \c\L.
  
 proc parseInteger(r: TRedis): TRedisInteger =
-  var line = r.socket.recv()
+  var line = r.socket.recv.string

  if line[0] == '-':
    raise newException(ERedis, strip(line))
@@ -70,14 +70,14 @@ proc parseInteger(r: TRedis): TRedisInteger =
  if parseBiggestInt(line, result, 1) == 0:
    raise newException(EInvalidReply, "Unable to parse integer.") 

-proc recv(sock: TSocket, size: int): string =
-  result = newString(size)
+proc recv(sock: TSocket, size: int): TaintedString =
+  result = newString(size).TaintedString
  if sock.recv(cstring(result), size) != size:
    raise newException(EInvalidReply, "recv failed")

 proc parseBulk(r: TRedis, allowMBNil = False): TRedisString =
  var line = ""
-  if not r.socket.recvLine(line):
+  if not r.socket.recvLine(line.TaintedString):
    raise newException(EInvalidReply, "recvLine failed")
  
  # Error.
@@ -97,17 +97,17 @@ proc parseBulk(r: TRedis, allowMBNil = False): TRedisString =
    return RedisNil

  var s = r.socket.recv(numBytes+2)
-  result = strip(s)
+  result = strip(s.string)

 proc parseMultiBulk(r: TRedis): TRedisList =
-  var line = ""
+  var line = TaintedString""
  if not r.socket.recvLine(line):
    raise newException(EInvalidReply, "recvLine failed")
    
-  if line[0] != '*':
-    raiseInvalidReply('*', line[0])
+  if line.string[0] != '*':
+    raiseInvalidReply('*', line.string[0])
  
-  var numElems = parseInt(line.substr(1))
+  var numElems = parseInt(line.string.substr(1))
  if numElems == -1: return nil
  result = @[]
  for i in 1..numElems:
@@ -839,7 +839,7 @@ proc shutdown*(r: TRedis) =
  ## Synchronously save the dataset to disk and then shut down the server
  r.sendCommand("SHUTDOWN")
  var s = r.socket.recv()
-  if s != "": raise newException(ERedis, s)
+  if s.string.len != 0: raise newException(ERedis, s.string)

 proc slaveof*(r: TRedis, host: string, port: string) =
  ## Make the server a slave of another instance, or promote it as master
--- a/lib/pure/smtp.nim
+++ b/lib/pure/smtp.nim
@@ -59,8 +59,8 @@ proc debugSend(smtp: TSMTP, cmd: string) =
    when not defined(noSSL):
      smtp.sslSock.send(cmd)

-proc debugRecv(smtp: TSMTP): string =
-  var line = ""
+proc debugRecv(smtp: TSMTP): TaintedString =
+  var line = TaintedString""
  var ret = False
  if not smtp.ssl:
    ret = smtp.sock.recvLine(line)
@@ -69,11 +69,11 @@ proc debugRecv(smtp: TSMTP): string =
      ret = smtp.sslSock.recvLine(line)
  if ret:
    if smtp.debug:
-      echo("S:" & line)
+      echo("S:" & line.string)
    return line
  else:
    OSError()
-    return ""
+    return TaintedString""

 proc quitExcpt(smtp: TSMTP, msg: string) =
  smtp.debugSend("QUIT")
@@ -81,8 +81,8 @@ proc quitExcpt(smtp: TSMTP, msg: string) =

 proc checkReply(smtp: TSMTP, reply: string) =
  var line = smtp.debugRecv()
-  if not line.startswith(reply):
-    quitExcpt(smtp, "Expected " & reply & " reply, got: " & line)
+  if not line.string.startswith(reply):
+    quitExcpt(smtp, "Expected " & reply & " reply, got: " & line.string)

 proc connect*(address: string, port = 25, 
              ssl = false, debug = false): TSMTP =
--- a/lib/pure/sockets.nim
+++ b/lib/pure/sockets.nim
@@ -553,17 +553,28 @@ proc recv*(socket: TSocket): TaintedString =
  ## If socket is not a connectionless socket and socket is not connected
  ## ``""`` will be returned.
  const bufSize = 1000
-  var buf = newString(bufSize)
-  result = TaintedString""
+  result = newStringOfCap(bufSize).TaintedString
+  var pos = 0
  while true:
-    var bytesRead = recv(socket, cstring(buf), bufSize-1)
-    # Error
+    var bytesRead = recv(socket, addr(string(result)[pos]), bufSize-1)
    if bytesRead == -1: OSError()
-    
-    buf[bytesRead] = '\0' # might not be necessary
-    setLen(buf, bytesRead)
-    add(result.string, buf)
+    setLen(result.string, pos + bytesRead)
    if bytesRead != bufSize-1: break
+    # increase capacity:
+    setLen(result.string, result.string.len + bufSize)
+    inc(pos, bytesRead)
+  when false:
+    var buf = newString(bufSize)
+    result = TaintedString""
+    while true:
+      var bytesRead = recv(socket, cstring(buf), bufSize-1)
+      # Error
+      if bytesRead == -1: OSError()
+      
+      buf[bytesRead] = '\0' # might not be necessary
+      setLen(buf, bytesRead)
+      add(result.string, buf)
+      if bytesRead != bufSize-1: break

 proc recvAsync*(socket: TSocket, s: var TaintedString): bool =
  ## receives all the data from a non-blocking socket. If socket is non-blocking 
@@ -572,11 +583,12 @@ proc recvAsync*(socket: TSocket, s: var TaintedString): bool =
  ## If socket is not a connectionless socket and socket is not connected
  ## ``s`` will be set to ``""``.
  const bufSize = 1000
-  var buf = newString(bufSize)
-  s = ""
+  # ensure bufSize capacity:
+  setLen(s.string, bufSize)
+  setLen(s.string, 0)
+  var pos = 0
  while true:
-    var bytesRead = recv(socket, cstring(buf), bufSize-1)
-    # Error
+    var bytesRead = recv(socket, addr(string(s)[pos]), bufSize-1)
    if bytesRead == -1:
      when defined(windows):
        # TODO: Test on Windows
@@ -588,11 +600,12 @@ proc recvAsync*(socket: TSocket, s: var TaintedString): bool =
        if errno == EAGAIN or errno == EWOULDBLOCK:
          return False
        else: OSError()
-    
-    buf[bytesRead] = '\0' # might not be necessary
-    setLen(buf, bytesRead)
-    add(s, buf)
+
+    setLen(s.string, pos + bytesRead)
    if bytesRead != bufSize-1: break
+    # increase capacity:
+    setLen(s.string, s.string.len + bufSize)
+    inc(pos, bytesRead)
  result = True
  
 proc skip*(socket: TSocket) =
--- a/todo.txt
+++ b/todo.txt
@@ -1,7 +1,6 @@
 Version 0.8.14
 ==============

- taint mode
 - 'let x = y'; const ptr/ref
 - fix actors.nim
 - make threadvar efficient again on linux after testing
--- a/web/news.txt
+++ b/web/news.txt
@@ -61,6 +61,10 @@ Compiler Additions
  and Objective C somewhat easier.
 - Added a ``--nimcache:PATH`` configuration option for control over the output
  directory for generated code.
+- The compiler and standard library now support a *taint mode*. Input strings
+  are declared with the ``TaintedString`` string type. If the taint
+  mode is turned on it is a distinct string type which helps to detect input
+  validation errors.


 Library Additions