Improvements to httpclient. Refs #4423.

* Adds ability to query HttpCode and compare it with strings. * Moves HttpMethod to HttpCore module. * Implements synchronous HttpClient using {.multisync.}.
2026-01-07 13:33:22 +00:00 · 2016-09-18 18:16:51 +02:00
parent 1740619c0c
commit 3ad368f8ca
4 changed files with 198 additions and 87 deletions
--- a/lib/pure/httpclient.nim
+++ b/lib/pure/httpclient.nim
@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2010 Dominik Picheta, Andreas Rumpf
+#        (c) Copyright 2016 Dominik Picheta, Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -87,12 +87,20 @@ import nativesockets
 export httpcore except parseHeader # TODO: The ``except`` doesn't work

 type
-  Response* = tuple[
-    version: string,
-    status: string,
-    headers: HttpHeaders,
-    body: string]
+  Response* = object
+    version*: string
+    status*: string
+    headers*: HttpHeaders
+    body*: string

+proc code*(response: Response): HttpCode {.raises: [ValueError].} =
+  ## Retrieves the specified response's ``HttpCode``.
+  ##
+  ## Raises a ``ValueError`` if the response's ``status`` does not have a
+  ## corresponding ``HttpCode``.
+  return parseEnum[HttpCode](response.status)
+
+type
  Proxy* = ref object
    url*: Uri
    auth*: string
@@ -253,25 +261,6 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response =
  else:
    result.body = ""

-type
-  HttpMethod* = enum  ## the requested HttpMethod
-    httpHEAD,         ## Asks for the response identical to the one that would
-                      ## correspond to a GET request, but without the response
-                      ## body.
-    httpGET,          ## Retrieves the specified resource.
-    httpPOST,         ## Submits data to be processed to the identified
-                      ## resource. The data is included in the body of the
-                      ## request.
-    httpPUT,          ## Uploads a representation of the specified resource.
-    httpDELETE,       ## Deletes the specified resource.
-    httpTRACE,        ## Echoes back the received request, so that a client
-                      ## can see what intermediate servers are adding or
-                      ## changing in the request.
-    httpOPTIONS,      ## Returns the HTTP methods that the server supports
-                      ## for specified address.
-    httpCONNECT       ## Converts the request connection to a transparent
-                      ## TCP/IP tunnel, usually used for proxies.
-
 {.deprecated: [THttpMethod: HttpMethod].}

 when not defined(ssl):
@@ -397,7 +386,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
  ## server takes longer than specified an ETimeout exception will be raised.
  var r = if proxy == nil: parseUri(url) else: proxy.url
  var hostUrl = if proxy == nil: r else: parseUri(url)
-  var headers = substr(httpMethod, len("http"))
+  var headers = substr(httpMethod, len("http")).toUpper()
  # TODO: Use generateHeaders further down once it supports proxies.

  var s = newSocket()
@@ -620,7 +609,7 @@ proc downloadFile*(url: string, outputFilename: string,
 proc generateHeaders(r: Uri, httpMethod: string,
                     headers: StringTableRef, body: string): string =
  # TODO: Use this in the blocking HttpClient once it supports proxies.
-  result = substr(httpMethod, len("http"))
+  result = substr(httpMethod, len("http")).toUpper()
  # TODO: Proxies
  result.add ' '
  if r.path[0] != '/': result.add '/'
@@ -643,8 +632,8 @@ proc generateHeaders(r: Uri, httpMethod: string,
  add(result, "\c\L")

 type
-  AsyncHttpClient* = ref object
-    socket: AsyncSocket
+  HttpClientBase*[SocketType] = ref object
+    socket: SocketType
    connected: bool
    currentURL: Uri ## Where we are currently connected.
    headers*: StringTableRef
@@ -653,6 +642,30 @@ type
    when defined(ssl):
      sslContext: net.SslContext

+type
+  HttpClient* = HttpClientBase[Socket]
+
+proc newHttpClient*(userAgent = defUserAgent,
+    maxRedirects = 5, sslContext = defaultSslContext): HttpClient =
+  ## Creates a new HttpClient instance.
+  ##
+  ## ``userAgent`` specifies the user agent that will be used when making
+  ## requests.
+  ##
+  ## ``maxRedirects`` specifies the maximum amount of redirects to follow,
+  ## default is 5.
+  ##
+  ## ``sslContext`` specifies the SSL context to use for HTTPS requests.
+  new result
+  result.headers = newStringTable(modeCaseInsensitive)
+  result.userAgent = userAgent
+  result.maxRedirects = maxRedirects
+  when defined(ssl):
+    result.sslContext = sslContext
+
+type
+  AsyncHttpClient* = HttpClientBase[AsyncSocket]
+
 {.deprecated: [PAsyncHttpClient: AsyncHttpClient].}

 proc newAsyncHttpClient*(userAgent = defUserAgent,
@@ -673,13 +686,14 @@ proc newAsyncHttpClient*(userAgent = defUserAgent,
  when defined(ssl):
    result.sslContext = sslContext

-proc close*(client: AsyncHttpClient) =
+proc close*(client: HttpClient | AsyncHttpClient) =
  ## Closes any connections held by the HTTP client.
  if client.connected:
    client.socket.close()
    client.connected = false

-proc recvFull(socket: AsyncSocket, size: int): Future[string] {.async.} =
+proc recvFull(socket: Socket | AsyncSocket,
+              size: int): Future[string] {.multisync.} =
  ## Ensures that all the data requested is read and returned.
  result = ""
  while true:
@@ -688,7 +702,8 @@ proc recvFull(socket: AsyncSocket, size: int): Future[string] {.async.} =
    if data == "": break # We've been disconnected.
    result.add data

-proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} =
+proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string]
+                 {.multisync.} =
  result = ""
  while true:
    var chunkSize = 0
@@ -721,9 +736,9 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} =
    # Trailer headers will only be sent if the request specifies that we want
    # them: http://tools.ietf.org/html/rfc2616#section-3.6.1

-proc parseBody(client: AsyncHttpClient,
+proc parseBody(client: HttpClient | AsyncHttpClient,
               headers: HttpHeaders,
-               httpVersion: string): Future[string] {.async.} =
+               httpVersion: string): Future[string] {.multisync.} =
  result = ""
  if headers.getOrDefault"Transfer-Encoding" == "chunked":
    result = await parseChunks(client)
@@ -752,8 +767,8 @@ proc parseBody(client: AsyncHttpClient,
          if buf == "": break
          result.add(buf)

-proc parseResponse(client: AsyncHttpClient,
-                   getBody: bool): Future[Response] {.async.} =
+proc parseResponse(client: HttpClient | AsyncHttpClient,
+                   getBody: bool): Future[Response] {.multisync.} =
  var parsedStatus = false
  var linei = 0
  var fullyRead = false
@@ -803,11 +818,17 @@ proc parseResponse(client: AsyncHttpClient,
  else:
    result.body = ""

-proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} =
+proc newConnection(client: HttpClient | AsyncHttpClient,
+                   url: Uri) {.multisync.} =
  if client.currentURL.hostname != url.hostname or
      client.currentURL.scheme != url.scheme:
    if client.connected: client.close()
-    client.socket = newAsyncSocket()
+
+    when client is HttpClient:
+      client.socket = newSocket()
+    elif client is AsyncHttpClient:
+      client.socket = newAsyncSocket()
+    else: {.fatal: "Unsupported client type".}

    # TODO: I should be able to write 'net.Port' here...
    let port =
@@ -829,8 +850,8 @@ proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} =
    client.currentURL = url
    client.connected = true

-proc request*(client: AsyncHttpClient, url: string, httpMethod: string,
-              body = ""): Future[Response] {.async.} =
+proc request*(client: HttpClient | AsyncHttpClient, url: string,
+              httpMethod: string, body = ""): Future[Response] {.multisync.} =
  ## Connects to the hostname specified by the URL and performs a request
  ## using the custom method string specified by ``httpMethod``.
  ##
@@ -853,8 +874,8 @@ proc request*(client: AsyncHttpClient, url: string, httpMethod: string,

  result = await parseResponse(client, httpMethod != "httpHEAD")

-proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET,
-              body = ""): Future[Response] =
+proc request*(client: HttpClient | AsyncHttpClient, url: string,
+              httpMethod = HttpGET, body = ""): Future[Response] {.multisync.} =
  ## Connects to the hostname specified by the URL and performs a request
  ## using the method specified.
  ##
@@ -863,9 +884,10 @@ proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET,
  ## connection can be closed by using the ``close`` procedure.
  ##
  ## The returned future will complete once the request is completed.
-  result = request(client, url, $httpMethod, body)
+  result = await request(client, url, $httpMethod, body)

-proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} =
+proc get*(client: HttpClient | AsyncHttpClient,
+          url: string): Future[Response] {.multisync.} =
  ## Connects to the hostname specified by the URL and performs a GET request.
  ##
  ## This procedure will follow redirects up to a maximum number of redirects
@@ -878,7 +900,8 @@ proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} =
      result = await client.request(redirectTo, httpGET)
      lastURL = redirectTo

-proc post*(client: AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[Response] {.async.} =
+proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "",
+           multipart: MultipartData = nil): Future[Response] {.multisync.} =
  ## Connects to the hostname specified by the URL and performs a POST request.
  ##
  ## This procedure will follow redirects up to a maximum number of redirects
@@ -895,45 +918,4 @@ proc post*(client: AsyncHttpClient, url: string, body = "", multipart: Multipart
    client.headers["Content-Type"] = mpHeader.split(": ")[1]
  client.headers["Content-Length"] = $len(xb)

-  result = await client.request(url, httpPOST, xb)
-
-when not defined(testing) and isMainModule:
-  when true:
-    # Async
-    proc main() {.async.} =
-      var client = newAsyncHttpClient()
-      var resp = await client.request("http://picheta.me")
-
-      echo("Got response: ", resp.status)
-      echo("Body:\n")
-      echo(resp.body)
-
-      resp = await client.request("http://picheta.me/asfas.html")
-      echo("Got response: ", resp.status)
-
-      resp = await client.request("http://picheta.me/aboutme.html")
-      echo("Got response: ", resp.status)
-
-      resp = await client.request("http://nim-lang.org/")
-      echo("Got response: ", resp.status)
-
-      resp = await client.request("http://nim-lang.org/download.html")
-      echo("Got response: ", resp.status)
-
-    waitFor main()
-
-  else:
-    #downloadFile("http://force7.de/nim/index.html", "nimindex.html")
-    #downloadFile("http://www.httpwatch.com/", "ChunkTest.html")
-    #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com",
-    # "validator.html")
-
-    #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com&
-    #  charset=%28detect+automatically%29&doctype=Inline&group=0")
-
-    var data = newMultipartData()
-    data["output"] = "soap12"
-    data["uploaded_file"] = ("test.html", "text/html",
-      "<html><head></head><body><p>test</p></body></html>")
-
-    echo postContent("http://validator.w3.org/check", multipart=data)
+  result = await client.request(url, HttpPOST, xb)
--- a/lib/pure/httpcore.nim
+++ b/lib/pure/httpcore.nim
@@ -71,6 +71,28 @@ type
    HttpVer11,
    HttpVer10

+  HttpMethod* = enum  ## the requested HttpMethod
+    HttpHead,         ## Asks for the response identical to the one that would
+                      ## correspond to a GET request, but without the response
+                      ## body.
+    HttpGet,          ## Retrieves the specified resource.
+    HttpPost,         ## Submits data to be processed to the identified
+                      ## resource. The data is included in the body of the
+                      ## request.
+    HttpPut,          ## Uploads a representation of the specified resource.
+    HttpDelete,       ## Deletes the specified resource.
+    HttpTrace,        ## Echoes back the received request, so that a client
+                      ## can see what intermediate servers are adding or
+                      ## changing in the request.
+    HttpOptions,      ## Returns the HTTP methods that the server supports
+                      ## for specified address.
+    HttpConnect       ## Converts the request connection to a transparent
+                      ## TCP/IP tunnel, usually used for proxies.
+
+{.deprecated: [httpGet: HttpGet, httpHead: HttpHead, httpPost: HttpPost,
+               httpPut: HttpPut, httpDelete: HttpDelete, httpTrace: HttpTrace,
+               httpOptions: HttpOptions, httpConnect: HttpConnect].}
+
 const headerLimit* = 10_000

 proc newHttpHeaders*(): HttpHeaders =
@@ -188,6 +210,25 @@ proc `==`*(protocol: tuple[orig: string, major, minor: int],
    of HttpVer10: 0
  result = protocol.major == major and protocol.minor == minor

+proc `==`*(rawCode: string, code: HttpCode): bool =
+  return rawCode.toLower() == ($code).toLower()
+
+proc is2xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 2xx HTTP status code.
+  return ($code).startsWith("2")
+
+proc is3xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 3xx HTTP status code.
+  return ($code).startsWith("3")
+
+proc is4xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 4xx HTTP status code.
+  return ($code).startsWith("4")
+
+proc is5xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 5xx HTTP status code.
+  return ($code).startsWith("5")
+
 when isMainModule:
  var test = newHttpHeaders()
  test["Connection"] = @["Upgrade", "Close"]
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -966,6 +966,22 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1,
    socket.socketError(result, lastError = lastError)
  data.setLen(result)

+proc recv*(socket: Socket, size: int, timeout = -1,
+           flags = {SocketFlag.SafeDisconn}): string {.inline.} =
+  ## Higher-level version of ``recv`` which returns a string.
+  ##
+  ## When ``""`` is returned the socket's connection has been closed.
+  ##
+  ## This function will throw an EOS exception when an error occurs.
+  ##
+  ## A timeout may be specified in milliseconds, if enough data is not received
+  ## within the time specified an ETimeout exception will be raised.
+  ##
+  ##
+  ## **Warning**: Only the ``SafeDisconn`` flag is currently supported.
+  result = newString(size)
+  discard recv(socket, result, size, timeout, flags)
+
 proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} =
  if socket.isBuffered:
    result = 1
@@ -1035,6 +1051,25 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1,
      return
    add(line.string, c)

+proc recvLine*(socket: Socket, timeout = -1,
+               flags = {SocketFlag.SafeDisconn}): TaintedString =
+  ## Reads a line of data from ``socket``.
+  ##
+  ## If a full line is read ``\r\L`` is not
+  ## added to the result, however if solely ``\r\L`` is read then the result
+  ## will be set to it.
+  ##
+  ## If the socket is disconnected, the result will be set to ``""``.
+  ##
+  ## An EOS exception will be raised in the case of a socket error.
+  ##
+  ## A timeout can be specified in milliseconds, if data is not received within
+  ## the specified time an ETimeout exception will be raised.
+  ##
+  ## **Warning**: Only the ``SafeDisconn`` flag is currently supported.
+  result = ""
+  readLine(socket, result, timeout, flags)
+
 proc recvFrom*(socket: Socket, data: var string, length: int,
               address: var string, port: var Port, flags = 0'i32): int {.
               tags: [ReadIOEffect].} =
--- a/tests/stdlib/thttpclient.nim
+++ b/tests/stdlib/thttpclient.nim
@@ -0,0 +1,53 @@
+import strutils
+
+import httpclient, asyncdispatch
+
+proc asyncTest() {.async.} =
+  var client = newAsyncHttpClient()
+  var resp = await client.request("http://example.com/")
+  doAssert(resp.code.is2xx)
+  doAssert("<title>Example Domain</title>" in resp.body)
+
+  resp = await client.request("http://example.com/404")
+  doAssert(resp.code.is4xx)
+  doAssert(resp.code == Http404)
+  doAssert(resp.status == Http404)
+
+  resp = await client.request("https://google.com/")
+  doAssert(resp.code.is2xx or resp.code.is3xx)
+
+proc syncTest() =
+  var client = newHttpClient()
+  var resp = client.request("http://example.com/")
+  doAssert(resp.code.is2xx)
+  doAssert("<title>Example Domain</title>" in resp.body)
+
+  resp = client.request("http://example.com/404")
+  doAssert(resp.code.is4xx)
+  doAssert(resp.code == Http404)
+  doAssert(resp.status == Http404)
+
+  resp = client.request("https://google.com/")
+  doAssert(resp.code.is2xx or resp.code.is3xx)
+
+syncTest()
+
+waitFor(asyncTest())
+
+#[
+
+  else:
+    #downloadFile("http://force7.de/nim/index.html", "nimindex.html")
+    #downloadFile("http://www.httpwatch.com/", "ChunkTest.html")
+    #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com",
+    # "validator.html")
+
+    #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com&
+    #  charset=%28detect+automatically%29&doctype=Inline&group=0")
+
+    var data = newMultipartData()
+    data["output"] = "soap12"
+    data["uploaded_file"] = ("test.html", "text/html",
+      "<html><head></head><body><p>test</p></body></html>")
+
+    echo postContent("http://validator.w3.org/check", multipart=data)]#