Merge branch 'master' of github.com:Araq/Nimrod into upstream

lib/pure/httpclient.nim

@@ -87,10 +87,11 @@ proc charAt(d: var string, i: var int, s: TSocket): char {.inline.} =
     i = 0
     result = d[i]
 
-proc parseChunks(d: var string, start: int, s: TSocket): string =
+proc parseChunks(s: TSocket): string =
   # get chunks:
-  var i = start
+  var i = 0
   result = ""
+  var d = s.recv().string
   while true:
     var chunkSize = 0
     var digitFound = false
@@ -136,88 +137,87 @@ proc parseChunks(d: var string, start: int, s: TSocket): string =
     # skip trailing CR-LF:
     while charAt(d, i, s) in {'\C', '\L'}: inc(i)
   
-proc parseBody(d: var string, start: int, s: TSocket,
+proc parseBody(s: TSocket,
                headers: PStringTable): string =
+  result = ""
   if headers["Transfer-Encoding"] == "chunked":
-    result = parseChunks(d, start, s)
+    result = parseChunks(s)
   else:
-    result = substr(d, start)
     # -REGION- Content-Length
     # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3
     var contentLengthHeader = headers["Content-Length"]
     if contentLengthHeader != "":
       var length = contentLengthHeader.parseint()
-      while result.len() < length: result.add(s.recv.string)
+      result = newString(length)
+      var received = 0
+      while true:
+        if received >= length: break
+        let r = s.recv(addr(result[received]), length-received)
+        if r == 0: break
+        received += r
+      if received != length:
+        httpError("Got invalid content length. Expected: " & $length &
+                  " got: " & $received)
     else:
       # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO
       
       # -REGION- Connection: Close
       # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
       if headers["Connection"] == "close":
+        var buf = ""
         while True:
-          var moreData = recv(s).string
-          if moreData.len == 0: break
-          result.add(moreData)
+          buf = newString(4000)
+          let r = s.recv(addr(buf[0]), 4000)
+          if r == 0: break
+          buf.setLen(r)
+          result.add(buf)
 
-proc parseResponse(s: TSocket): TResponse =
-  var d = s.recv.string # Warning: without a Connection: Close header this will not work.
-  var i = 0
-
-  # Parse the version
-  # Parses the first line of the headers
-  # ``HTTP/1.1`` 200 OK
-  var L = skipIgnoreCase(d, "HTTP/1.1", i)
-  if L > 0:
-    result.version = "1.1"
-    inc(i, L)
-  else:
-    L = skipIgnoreCase(d, "HTTP/1.0", i)
-    if L > 0:
-      result.version = "1.0"
-      inc(i, L)
-    else: 
-      httpError("invalid HTTP header")
-  L = skipWhiteSpace(d, i)
-  if L <= 0: httpError("invalid HTTP header")
-  inc(i, L)
-  
-  result.status = ""
-  while d[i] notin {'\C', '\L', '\0'}:
-    result.status.add(d[i])
-    inc(i)
-  if d[i] == '\C': inc(i)
-  if d[i] == '\L': inc(i)
-  else: httpError("invalid HTTP header, CR-LF expected")
-
-  # Parse the headers
-  # Everything after the first line leading up to the body
-  # htype: hvalue
+proc parseResponse(s: TSocket, getBody: bool): TResponse =
+  var parsedStatus = false
+  var linei = 0
+  var fullyRead = false
+  var line = ""
   result.headers = newStringTable(modeCaseInsensitive)
-  while true:
-    var key = ""
-    while d[i] != ':':
-      if d[i] == '\0': httpError("invalid HTTP header, ':' expected")
-      key.add(d[i])
-      inc(i)
-    inc(i) # skip ':'
-    if d[i] == ' ': inc(i) # skip if the character is a space
-    var val = ""
-    while d[i] notin {'\C', '\L', '\0'}:
-      val.add(d[i])
-      inc(i)
-    
-    result.headers[key] = val
-    
-    if d[i] == '\C': inc(i)
-    if d[i] == '\L': inc(i)
-    else: httpError("invalid HTTP header, CR-LF expected")
-    
-    if d[i] == '\C': inc(i)
-    if d[i] == '\L':
-      inc(i)
-      break
-    
-  result.body = parseBody(d, i, s, result.headers) 
+  while True:
+    line = ""
+    linei = 0
+    if s.recvLine(line):
+      if line == "": break # We've been disconnected.
+      if line == "\c\L":
+        fullyRead = true
+        break
+      if not parsedStatus:
+        # Parse HTTP version info and status code.
+        var le = skipIgnoreCase(line, "HTTP/", linei)
+        if le <= 0: httpError("invalid http version")
+        inc(linei, le)
+        le = skipIgnoreCase(line, "1.1", linei)
+        if le > 0: result.version = "1.1"
+        else:
+          le = skipIgnoreCase(line, "1.0", linei)
+          if le <= 0: httpError("unsupported http version")
+          result.version = "1.0"
+        inc(linei, le)
+        # Status code
+        linei.inc skipWhitespace(line, linei)
+        result.status = line[linei .. -1]
+        parsedStatus = true
+      else:
+        # Parse headers
+        var name = ""
+        var le = parseUntil(line, name, ':', linei)
+        if le <= 0: httpError("invalid headers")
+        inc(linei, le)
+        if line[linei] != ':': httpError("invalid headers")
+        inc(linei) # Skip :
+        linei += skipWhitespace(line, linei)
+        
+        result.headers[name] = line[linei.. -1]
+  if not fullyRead: httpError("Connection was closed before full request has been made")
+  if getBody:
+    result.body = parseBody(s, result.headers)
+  else:
+    result.body = ""
 
 type
   THttpMethod* = enum ## the requested HttpMethod
@@ -246,10 +246,10 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
   
   var headers = substr($httpMethod, len("http"))
   headers.add(" /" & r.path & r.query)
+
   headers.add(" HTTP/1.1\c\L")
   
   add(headers, "Host: " & r.hostname & "\c\L")
-  add(headers, "Connection: Close\c\L")
   add(headers, extraHeaders)
   add(headers, "\c\L")
 
@@ -258,6 +258,8 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
   if r.scheme == "https":
     when defined(ssl):
       s.wrapSocket(verifyMode = CVerifyNone)
+    else:
+      raise newException(EHttpRequestErr, "SSL support was not compiled in. Cannot connect over SSL.")
     port = TPort(443)
   if r.port != "":
     port = TPort(r.port.parseInt)
@@ -266,7 +268,7 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
   if body != "":
     s.send(body)
   
-  result = parseResponse(s)
+  result = parseResponse(s, httpMethod != httpHEAD)
   s.close()
   
 proc redirection(status: string): bool =

lib/pure/sockets.nim

@@ -407,7 +407,7 @@ proc acceptAddr*(server: TSocket): tuple[client: TSocket, address: string] =
     when defined(windows):
       var err = WSAGetLastError()
       if err == WSAEINPROGRESS:
-        client = InvalidSocket
+        return (InvalidSocket, "")
       else: OSError()
     else:
       if errno == EAGAIN or errno == EWOULDBLOCK:
@@ -775,15 +775,18 @@ proc readIntoBuf(socket: TSocket, flags: int32): int =
       result = recv(socket.fd, addr(socket.buffer), int(socket.buffer.high), flags)
   else:
     result = recv(socket.fd, addr(socket.buffer), int(socket.buffer.high), flags)
-  if result <= 0: return
+  if result <= 0:
+    socket.buflen = 0
+    socket.currPos = 0
+    return result
   socket.bufLen = result
   socket.currPos = 0
 
-template retRead(flags, read: int) =
+template retRead(flags, readBytes: int) =
   let res = socket.readIntoBuf(flags)
   if res <= 0:
-    if read > 0:
-      return read
+    if readBytes > 0:
+      return readBytes
     else:
       return res
 
@@ -793,16 +796,27 @@ proc recv*(socket: TSocket, data: pointer, size: int): int =
     if socket.bufLen == 0:
       retRead(0'i32, 0)
     
-    var read = 0
-    while read < size:
-      if socket.currPos >= socket.bufLen:
-        retRead(0'i32, read)
-    
-      let chunk = min(socket.bufLen, size-read)
-      var d = cast[cstring](data)
-      copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk)
-      read.inc(chunk)
-      socket.currPos.inc(chunk)
+    when true:
+      var read = 0
+      while read < size:
+        if socket.currPos >= socket.bufLen:
+          retRead(0'i32, read)
+      
+        let chunk = min(socket.bufLen-socket.currPos, size-read)
+        var d = cast[cstring](data)
+        copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk)
+        read.inc(chunk)
+        socket.currPos.inc(chunk)
+    else:
+      var read = 0
+      while read < size:
+        if socket.currPos >= socket.bufLen:
+          retRead(0'i32, read)
+      
+        var d = cast[cstring](data)
+        d[read] = socket.buffer[socket.currPos]
+        read.inc(1)
+        socket.currPos.inc(1)
     
     result = read
   else:
@@ -814,8 +828,14 @@ proc recv*(socket: TSocket, data: pointer, size: int): int =
     else:
       result = recv(socket.fd, data, size, 0'i32)
 
-proc waitFor(socket: TSocket, waited: var float, timeout: int) =
-  if socket.bufLen == 0:
+proc waitFor(socket: TSocket, waited: var float, timeout: int): int =
+  ## returns the number of characters available to be read. In unbuffered
+  ## sockets this is always 1, otherwise this may as big as the buffer, currently
+  ## 4000.
+  result = 1
+  if socket.isBuffered and socket.bufLen != 0 and socket.bufLen != socket.currPos:
+    result = socket.bufLen - socket.currPos
+  else:
     if timeout - int(waited * 1000.0) < 1:
       raise newException(ETimeout, "Call to recv() timed out.")
     var s = @[socket]
@@ -824,17 +844,19 @@ proc waitFor(socket: TSocket, waited: var float, timeout: int) =
       raise newException(ETimeout, "Call to recv() timed out.")
     waited += (epochTime() - startTime)
 
-proc recv*(socket: TSocket, data: var string, size: int, timeout: int): int =
+proc recv*(socket: TSocket, data: pointer, size: int, timeout: int): int =
   ## overload with a ``timeout`` parameter in miliseconds.
   var waited = 0.0 # number of seconds already waited  
   
   var read = 0
   while read < size:
-    waitFor(socket, waited, timeout)
-    result = recv(socket, addr(data[read]), 1)
+    let avail = waitFor(socket, waited, timeout)
+    var d = cast[cstring](data)
+    result = recv(socket, addr(d[read]), avail)
+    if result == 0: break
     if result < 0:
-      return
-    inc(read)
+      return result
+    inc(read, result)
   
   result = read
 
@@ -901,12 +923,12 @@ proc recvLine*(socket: TSocket, line: var TaintedString, timeout: int): bool =
   setLen(line.string, 0)
   while true:
     var c: char
-    waitFor(socket, waited, timeout)
+    discard waitFor(socket, waited, timeout)
     var n = recv(socket, addr(c), 1)
     if n < 0: return
     elif n == 0: return true
     if c == '\r':
-      waitFor(socket, waited, timeout)
+      discard waitFor(socket, waited, timeout)
       n = peekChar(socket, c)
       if n > 0 and c == '\L':
         discard recv(socket, addr(c), 1)