From 860264a382a2e8cb44ea358fbbfcd1e763964097 Mon Sep 17 00:00:00 2001 From: Dmitry Polienko Date: Fri, 28 Oct 2016 17:34:07 +0700 Subject: [PATCH 1/3] Skip empty lines before status line As recommended here: https://tools.ietf.org/html/rfc7230#section-3.5 --- lib/pure/asynchttpserver.nim | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 4c7037a4e1..4438a5ff44 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -133,14 +133,21 @@ proc processClient(client: AsyncSocket, address: string, assert client != nil request.client = client - # First line - GET /path HTTP/1.1 - lineFut.mget().setLen(0) - lineFut.clean() - await client.recvLineInto(lineFut) # TODO: Timeouts. - if lineFut.mget == "": - client.close() - return + # We should skip empty lines before the request + # https://tools.ietf.org/html/rfc7230#section-3.5 + while true: + lineFut.mget().setLen(0) + lineFut.clean() + await client.recvLineInto(lineFut) # TODO: Timeouts. + if lineFut.mget == "": + client.close() + return + + if lineFut.mget != "\c\L": + break + + # First line - GET /path HTTP/1.1 var i = 0 for linePart in lineFut.mget.split(' '): case i From 52f4590e98832b536282fd2c90ca36246cc95b80 Mon Sep 17 00:00:00 2001 From: Dmitry Polienko Date: Mon, 31 Oct 2016 09:46:50 +0700 Subject: [PATCH 2/3] Don't persist generated headers between requests --- lib/pure/httpclient.nim | 45 ++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index c56d13b574..080229756a 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -117,7 +117,7 @@ ## only basic authentication is supported at the moment. import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, - math, random, httpcore, times + math, random, httpcore, times, tables import asyncnet, asyncdispatch import nativesockets @@ -989,8 +989,17 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.currentURL = url client.connected = true +proc withFallback(override, fallback: HttpHeaders): HttpHeaders = + # Left-biased map union for `HttpHeaders` + result = newHttpHeaders() + # Copy by value + result.table[] = fallback.table[] + for k, vs in override.table: + result[k] = vs + proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod: string, body = ""): Future[Response] {.multisync.} = + httpMethod: string, body = "", + overrideHeaders: HttpHeaders = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the custom method string specified by ``httpMethod``. ## @@ -1023,11 +1032,17 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, else: await newConnection(client, connectionUrl) - if not client.headers.hasKey("user-agent") and client.userAgent != "": - client.headers["User-Agent"] = client.userAgent + var effectiveHeaders: HttpHeaders + if overrideHeaders != nil: + effectiveHeaders = overrideHeaders.withFallback(client.headers) + else: + effectiveHeaders = client.headers + + if not effectiveHeaders.hasKey("user-agent") and client.userAgent != "": + effectiveHeaders["User-Agent"] = client.userAgent var headers = generateHeaders(requestUrl, httpMethod, - client.headers, body, client.proxy) + effectiveHeaders, body, client.proxy) await client.socket.send(headers) if body != "": @@ -1040,7 +1055,8 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, client.proxy = savedProxy proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod = HttpGET, body = ""): Future[Response] {.multisync.} = + httpMethod = HttpGET, body = "", + overrideHeaders: HttpHeaders = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the method specified. ## @@ -1050,7 +1066,8 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## ## When a request is made to a different hostname, the current connection will ## be closed. - result = await request(client, url, $httpMethod, body) + result = await request(client, url, $httpMethod, body, + overrideHeaders = overrideHeaders) proc get*(client: HttpClient | AsyncHttpClient, url: string): Future[Response] {.multisync.} = @@ -1097,18 +1114,22 @@ proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", else: x var xb = mpBody.withNewLine() & body - if multipart != nil: - client.headers["Content-Type"] = mpHeader.split(": ")[1] - client.headers["Content-Length"] = $len(xb) - result = await client.request(url, HttpPOST, xb) + var overrideHeaders = newHttpHeaders() + if multipart != nil: + overrideHeaders["Content-Type"] = mpHeader.split(": ")[1] + overrideHeaders["Content-Length"] = $len(xb) + + result = await client.request(url, HttpPOST, xb, + overrideHeaders = overrideHeaders) # Handle redirects. var lastURL = url for i in 1..client.maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) var meth = if result.status != "307": HttpGet else: HttpPost - result = await client.request(redirectTo, meth, xb) + result = await client.request(redirectTo, meth, xb, + overrideHeaders = overrideHeaders) lastURL = redirectTo proc postContent*(client: HttpClient | AsyncHttpClient, url: string, From 4753149227e9ca1402e59cc55870c56c8f872337 Mon Sep 17 00:00:00 2001 From: Dmitry Polienko Date: Wed, 2 Nov 2016 09:19:20 +0700 Subject: [PATCH 3/3] Implement suggestions from @dom96 --- lib/pure/asynchttpserver.nim | 4 ++-- lib/pure/httpclient.nim | 35 +++++++++++++++++------------------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 4438a5ff44..68da5f7c9d 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -133,9 +133,9 @@ proc processClient(client: AsyncSocket, address: string, assert client != nil request.client = client - # We should skip empty lines before the request + # We should skip at least one empty line before the request # https://tools.ietf.org/html/rfc7230#section-3.5 - while true: + for i in 0..1: lineFut.mget().setLen(0) lineFut.clean() await client.recvLineInto(lineFut) # TODO: Timeouts. diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 080229756a..ba967b14fc 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -989,8 +989,11 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.currentURL = url client.connected = true -proc withFallback(override, fallback: HttpHeaders): HttpHeaders = - # Left-biased map union for `HttpHeaders` +proc override(fallback, override: HttpHeaders): HttpHeaders = + # Right-biased map union for `HttpHeaders` + if override.isNil: + return fallback + result = newHttpHeaders() # Copy by value result.table[] = fallback.table[] @@ -999,7 +1002,7 @@ proc withFallback(override, fallback: HttpHeaders): HttpHeaders = proc request*(client: HttpClient | AsyncHttpClient, url: string, httpMethod: string, body = "", - overrideHeaders: HttpHeaders = nil): Future[Response] {.multisync.} = + headers: HttpHeaders = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the custom method string specified by ``httpMethod``. ## @@ -1032,19 +1035,15 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, else: await newConnection(client, connectionUrl) - var effectiveHeaders: HttpHeaders - if overrideHeaders != nil: - effectiveHeaders = overrideHeaders.withFallback(client.headers) - else: - effectiveHeaders = client.headers + let effectiveHeaders = client.headers.override(headers) if not effectiveHeaders.hasKey("user-agent") and client.userAgent != "": effectiveHeaders["User-Agent"] = client.userAgent - var headers = generateHeaders(requestUrl, httpMethod, - effectiveHeaders, body, client.proxy) + var headersString = generateHeaders(requestUrl, httpMethod, + effectiveHeaders, body, client.proxy) - await client.socket.send(headers) + await client.socket.send(headersString) if body != "": await client.socket.send(body) @@ -1056,7 +1055,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, proc request*(client: HttpClient | AsyncHttpClient, url: string, httpMethod = HttpGET, body = "", - overrideHeaders: HttpHeaders = nil): Future[Response] {.multisync.} = + headers: HttpHeaders = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the method specified. ## @@ -1067,7 +1066,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## When a request is made to a different hostname, the current connection will ## be closed. result = await request(client, url, $httpMethod, body, - overrideHeaders = overrideHeaders) + headers = headers) proc get*(client: HttpClient | AsyncHttpClient, url: string): Future[Response] {.multisync.} = @@ -1115,13 +1114,13 @@ proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", x var xb = mpBody.withNewLine() & body - var overrideHeaders = newHttpHeaders() + var headers = newHttpHeaders() if multipart != nil: - overrideHeaders["Content-Type"] = mpHeader.split(": ")[1] - overrideHeaders["Content-Length"] = $len(xb) + headers["Content-Type"] = mpHeader.split(": ")[1] + headers["Content-Length"] = $len(xb) result = await client.request(url, HttpPOST, xb, - overrideHeaders = overrideHeaders) + headers = headers) # Handle redirects. var lastURL = url for i in 1..client.maxRedirects: @@ -1129,7 +1128,7 @@ proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", let redirectTo = getNewLocation(lastURL, result.headers) var meth = if result.status != "307": HttpGet else: HttpPost result = await client.request(redirectTo, meth, xb, - overrideHeaders = overrideHeaders) + headers = headers) lastURL = redirectTo proc postContent*(client: HttpClient | AsyncHttpClient, url: string,