Rewrote the implementation of parsing chunked transfer coding in

httpclient. Fixes #272.

lib/pure/httpclient.nim

@@ -80,62 +80,43 @@ proc fileError(msg: string) =
   e.msg = msg
   raise e
 
-proc charAt(d: var string, i: var int, s: TSocket): char {.inline.} = 
-  result = d[i]
-  while result == '\0':
-    d = string(s.recv())
-    i = 0
-    result = d[i]
-
 proc parseChunks(s: TSocket): string =
-  # get chunks:
-  var i = 0
   result = ""
-  var d = s.recv().string
+  var ri = 0
   while true:
+    var chunkSizeStr = ""
     var chunkSize = 0
-    var digitFound = false
-    while true: 
-      case d[i]
-      of '0'..'9': 
-        digitFound = true
-        chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('0'))
-      of 'a'..'f': 
-        digitFound = true
-        chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('a') + 10)
-      of 'A'..'F': 
-        digitFound = true
-        chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('A') + 10)
-      of '\0': 
-        d = string(s.recv())
-        i = -1
-      else: break
-      inc(i)
-    if not digitFound: httpError("Chunksize expected")
+    if s.recvLine(chunkSizeStr):
+      var i = 0
+      if chunkSizeStr == "":
+        httpError("Server terminated connection prematurely")
+      while true:
+        case chunkSizeStr[i]
+        of '0'..'9':
+          chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0'))
+        of 'a'..'f':
+          chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10)
+        of 'A'..'F':
+          chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10)
+        of '\0':
+          break
+        of ';':
+          # http://tools.ietf.org/html/rfc2616#section-3.6.1
+          # We don't care about chunk-extensions.
+          break
+        else:
+          httpError("Invalid chunk size: " & chunkSizeStr)
+        inc(i)
     if chunkSize <= 0: break
-    while charAt(d, i, s) notin {'\C', '\L', '\0'}: inc(i)
-    if charAt(d, i, s) == '\C': inc(i)
-    if charAt(d, i, s) == '\L': inc(i)
-    else: httpError("CR-LF after chunksize expected")
-    
-    var x = substr(d, i, i+chunkSize-1)
-    var size = x.len
-    result.add(x)
-    inc(i, size)
-    if size < chunkSize:
-      # read in the rest:
-      var missing = chunkSize - size
-      var L = result.len
-      setLen(result, L + missing)    
-      while missing > 0:
-        var bytesRead = s.recv(addr(result[L]), missing)
-        inc(L, bytesRead)
-        dec(missing, bytesRead)
-      # next chunk:
-      d = string(s.recv())
-      i = 0
-    # skip trailing CR-LF:
-    while charAt(d, i, s) in {'\C', '\L'}: inc(i)
+    result.setLen(ri+chunkSize)
+    var bytesRead = 0
+    while bytesRead != chunkSize:
+      let ret = recv(s, addr(result[ri]), chunkSize-bytesRead)
+      ri += ret
+      bytesRead += ret
+    s.skip(2) # Skip \c\L
+    # Trailer headers will only be sent if the request specifies that we want
+    # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
   
 proc parseBody(s: TSocket,
                headers: PStringTable): string =
@@ -250,7 +231,6 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
   ## | Requests ``url`` with the specified ``httpMethod``.
   ## | Extra headers can be specified and must be seperated by ``\c\L``
   var r = parseUrl(url)
-  
   var headers = substr($httpMethod, len("http"))
   headers.add(" /" & r.path & r.query)
 
@@ -285,7 +265,7 @@ proc redirection(status: string): bool =
       return True
   
 proc get*(url: string, maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse =
-  ## | GET's the ``url`` and returns a ``TResponse`` object
+  ## | GETs the ``url`` and returns a ``TResponse`` object
   ## | This proc also handles redirection
   result = request(url)
   for i in 1..maxRedirects:
@@ -295,7 +275,7 @@ proc get*(url: string, maxRedirects = 5, sslContext: PSSLContext = defaultSSLCon
       result = request(locationHeader, sslContext = sslContext)
       
 proc getContent*(url: string, sslContext: PSSLContext = defaultSSLContext): string =
-  ## | GET's the body and returns it as a string.
+  ## | GETs the body and returns it as a string.
   ## | Raises exceptions for the status codes ``4xx`` and ``5xx``
   var r = get(url, sslContext = sslContext)
   if r.status[0] in {'4','5'}:
@@ -305,7 +285,7 @@ proc getContent*(url: string, sslContext: PSSLContext = defaultSSLContext): stri
   
 proc post*(url: string, extraHeaders = "", body = "", 
            maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse =
-  ## | POST's ``body`` to the ``url`` and returns a ``TResponse`` object.
+  ## | POSTs ``body`` to the ``url`` and returns a ``TResponse`` object.
   ## | This proc adds the necessary Content-Length header.
   ## | This proc also handles redirection.
   var xh = extraHeaders & "Content-Length: " & $len(body) & "\c\L"
@@ -319,7 +299,7 @@ proc post*(url: string, extraHeaders = "", body = "",
   
 proc postContent*(url: string, extraHeaders = "", body = "",
                   sslContext: PSSLContext = defaultSSLContext): string =
-  ## | POST's ``body`` to ``url`` and returns the response's body as a string
+  ## | POSTs ``body`` to ``url`` and returns the response's body as a string
   ## | Raises exceptions for the status codes ``4xx`` and ``5xx``
   var r = post(url, extraHeaders, body)
   if r.status[0] in {'4','5'}:

lib/pure/sockets.nim

@@ -1164,11 +1164,13 @@ proc recvLineAsync*(socket: TSocket,
     elif c == '\L': return RecvFullLine
     add(line.string, c)
 
-proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO].} =
+proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO], deprecated.} =
   ## receives all the available data from the socket.
   ## Socket errors will result in an ``EOS`` error.
   ## If socket is not a connectionless socket and socket is not connected
   ## ``""`` will be returned.
+  ##
+  ## **Deprecated since version 0.9.2**: This function is not safe for use.
   const bufSize = 4000
   result = newStringOfCap(bufSize).TaintedString
   var pos = 0
@@ -1194,10 +1196,12 @@ proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO].} =
       if bytesRead != bufSize-1: break
 
 proc recvTimeout*(socket: TSocket, timeout: int): TaintedString {.
-  tags: [FReadIO].} =
+  tags: [FReadIO], deprecated.} =
   ## overloaded variant to support a ``timeout`` parameter, the ``timeout``
   ## parameter specifies the amount of miliseconds to wait for data on the
   ## socket.
+  ##
+  ## **Deprecated since version 0.9.2**: This function is not safe for use.
   if socket.bufLen == 0:
     var s = @[socket]
     if s.select(timeout) != 1:
@@ -1298,13 +1302,25 @@ proc recvFromAsync*(socket: TSocket, data: var String, length: int,
         return False
       else: OSError()
 
-proc skip*(socket: TSocket) {.tags: [FReadIO].} =
+proc skip*(socket: TSocket) {.tags: [FReadIO], deprecated.} =
   ## skips all the data that is pending for the socket
+  ##
+  ## **Deprecated since version 0.9.2**: This function is not safe for use.
   const bufSize = 1000
   var buf = alloc(bufSize)
   while recv(socket, buf, bufSize) == bufSize: nil
   dealloc(buf)
 
+proc skip*(socket: TSocket, size: int) =
+  ## Skips ``size`` amount of bytes.
+  ##
+  ## Returns the number of skipped bytes.
+  var dummy = alloc(size)
+  var bytesSkipped = 0
+  while bytesSkipped != size:
+    bytesSkipped += recv(socket, dummy, size-bytesSkipped)
+  dealloc(dummy)
+
 proc send*(socket: TSocket, data: pointer, size: int): int {.
   tags: [FWriteIO].} =
   ## sends data to a socket.