Implement file streaming for httpclient's MultipartData (#12982)

* Add `uploadFile` to POST files by streaming them

* Use constant for \c\L

* Formatting

* Remove uploadFile

* Implement MultipartData file streaming

* Remove unnecessary var annotations

* Call string on TaintedStrings

Fixes #12789

* Move cl constant to httpcore

* Fix `request` inconsistencies

* Update documentaion

* Clean up

* Skip multipart formatting when there's 0 entries

* Remove extraneous `cl` from multipart formatting

* Update MultipartData `$` to match old behaviour

* Update comment

* Address comments
This commit is contained in:
Zed
2020-03-06 19:55:45 +01:00
committed by GitHub
parent 7ae0811818
commit e056298ceb
2 changed files with 207 additions and 142 deletions

View File

@@ -52,6 +52,19 @@
##
## echo client.postContent("http://validator.w3.org/check", multipart=data)
##
## To stream files from disk when performing the request, use ``addFiles``.
##
## **Note:** This will allocate a new ``Mimetypes`` database every time you call
## it, you can pass your own via the ``mimeDb`` parameter to avoid this.
##
## .. code-block:: Nim
## let mimes = newMimetypes()
## var client = newHttpClient()
## var data = newMultipartData()
## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes)
##
## echo client.postContent("http://validator.w3.org/check", multipart=data)
##
## You can also make post requests with custom headers.
## This example sets ``Content-Type`` to ``application/json``
## and uses a json object for the body
@@ -177,7 +190,7 @@
include "system/inclrtl"
import net, strutils, uri, parseutils, base64, os, mimetypes,
import net, strutils, uri, parseutils, base64, os, mimetypes, streams,
math, random, httpcore, times, tables, streams, std/monotimes
import asyncnet, asyncdispatch, asyncfile
import nativesockets
@@ -252,9 +265,18 @@ type
url*: Uri
auth*: string
MultipartEntry = object
name, content: string
case isFile: bool
of true:
filename, contentType: string
fileSize: int64
isStream: bool
else: discard
MultipartEntries* = openArray[tuple[name, content: string]]
MultipartData* = ref object
content: seq[string]
content: seq[MultipartEntry]
ProtocolError* = object of IOError ## exception that is raised when server
## does not conform to the implemented
@@ -278,7 +300,6 @@ proc fileError(msg: string) =
e.msg = msg
raise e
when not defined(ssl):
type SslContext = ref object
var defaultSslContext {.threadvar.}: SslContext
@@ -297,24 +318,28 @@ proc newProxy*(url: string, auth = ""): Proxy =
proc newMultipartData*: MultipartData =
## Constructs a new ``MultipartData`` object.
MultipartData(content: @[])
MultipartData()
proc `$`*(data: MultipartData): string {.since: (1, 1).} =
## convert MultipartData to string so it's human readable when echo
## see https://github.com/nim-lang/Nim/issues/11863
const prefixLen = "Content-Disposition: form-data; ".len
for pos, item in data.content:
result &= "------------------------------ "
result.addInt pos
result &= " ------------------------------\n"
result &= item[prefixLen .. item.high]
proc add*(p: var MultipartData, name, content: string, filename: string = "",
contentType: string = "") =
## Add a value to the multipart data. Raises a `ValueError` exception if
## `name`, `filename` or `contentType` contain newline characters.
const sep = "-".repeat(30)
for pos, entry in data.content:
result.add(sep & center($pos, 3) & sep)
result.add("\nname=\"" & entry.name & "\"")
if entry.isFile:
result.add("; filename=\"" & entry.filename & "\"\n")
result.add("Content-Type: " & entry.contentType)
result.add("\n\n" & entry.content & "\n")
proc add*(p: MultipartData, name, content: string, filename: string = "",
contentType: string = "", useStream = true) =
## Add a value to the multipart data.
##
## When ``useStream`` is ``false``, the file will be read into memory.
##
## Raises a ``ValueError`` exception if
## ``name``, ``filename`` or ``contentType`` contain newline characters.
if {'\c', '\L'} in name:
raise newException(ValueError, "name contains a newline character")
if {'\c', '\L'} in filename:
@@ -322,19 +347,22 @@ proc add*(p: var MultipartData, name, content: string, filename: string = "",
if {'\c', '\L'} in contentType:
raise newException(ValueError, "contentType contains a newline character")
var str = "Content-Disposition: form-data; name=\"" & name & "\""
if filename.len > 0:
str.add("; filename=\"" & filename & "\"")
str.add("\c\L")
if contentType.len > 0:
str.add("Content-Type: " & contentType & "\c\L")
str.add("\c\L" & content & "\c\L")
var entry = MultipartEntry(
name: name,
content: content,
isFile: filename.len > 0
)
p.content.add(str)
if entry.isFile:
entry.isStream = useStream
entry.filename = filename
entry.contentType = contentType
proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData
p.content.add(entry)
proc add*(p: MultipartData, xs: MultipartEntries): MultipartData
{.discardable.} =
## Add a list of multipart entries to the multipart data `p`. All values are
## Add a list of multipart entries to the multipart data ``p``. All values are
## added without a filename and without a content type.
##
## .. code-block:: Nim
@@ -344,70 +372,77 @@ proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData
result = p
proc newMultipartData*(xs: MultipartEntries): MultipartData =
## Create a new multipart data object and fill it with the entries `xs`
## Create a new multipart data object and fill it with the entries ``xs``
## directly.
##
## .. code-block:: Nim
## var data = newMultipartData({"action": "login", "format": "json"})
result = MultipartData(content: @[])
result.add(xs)
result = MultipartData()
for entry in xs:
result.add(entry.name, entry.content)
proc addFiles*(p: var MultipartData, xs: openArray[tuple[name, file: string]]):
MultipartData {.discardable.} =
## Add files to a multipart data object. The file will be opened from your
## disk, read and sent with the automatically determined MIME type. Raises an
## `IOError` if the file cannot be opened or reading fails. To manually
## specify file content, filename and MIME type, use `[]=` instead.
proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]],
mimeDb = newMimetypes(), useStream = true):
MultipartData {.discardable.} =
## Add files to a multipart data object. The files will be streamed from disk
## when the request is being made. When ``stream`` is ``false``, the files are
## instead read into memory, but beware this is very memory ineffecient even
## for small files. The MIME types will automatically be determined.
## Raises an ``IOError`` if the file cannot be opened or reading fails. To
## manually specify file content, filename and MIME type, use ``[]=`` instead.
##
## .. code-block:: Nim
## data.addFiles({"uploaded_file": "public/test.html"})
var m = newMimetypes()
for name, file in xs.items:
var contentType: string
let (_, fName, ext) = splitFile(file)
if ext.len > 0:
contentType = m.getMimetype(ext[1..ext.high], "")
p.add(name, readFile(file).string, fName & ext, contentType)
contentType = mimeDb.getMimetype(ext[1..ext.high], "")
let content = if useStream: file else: readFile(file).string
p.add(name, content, fName & ext, contentType, useStream = useStream)
result = p
proc `[]=`*(p: var MultipartData, name, content: string) =
## Add a multipart entry to the multipart data `p`. The value is added
proc `[]=`*(p: MultipartData, name, content: string) =
## Add a multipart entry to the multipart data ``p``. The value is added
## without a filename and without a content type.
##
## .. code-block:: Nim
## data["username"] = "NimUser"
p.add(name, content)
proc `[]=`*(p: var MultipartData, name: string,
proc `[]=`*(p: MultipartData, name: string,
file: tuple[name, contentType, content: string]) =
## Add a file to the multipart data `p`, specifying filename, contentType and
## content manually.
## Add a file to the multipart data ``p``, specifying filename, contentType
## and content manually.
##
## .. code-block:: Nim
## data["uploaded_file"] = ("test.html", "text/html",
## "<html><head></head><body><p>test</p></body></html>")
p.add(name, file.content, file.name, file.contentType)
p.add(name, file.content, file.name, file.contentType, useStream = false)
proc format(p: MultipartData): tuple[contentType, body: string] =
if p == nil or p.content.len == 0:
return ("", "")
# Create boundary that is not in the data to be formatted
var bound: string
proc getBoundary(p: MultipartData): string =
if p == nil or p.content.len == 0: return
while true:
bound = $random(int.high)
var found = false
for s in p.content:
if bound in s:
found = true
if not found:
break
result = $random(int.high)
for i, entry in p.content:
if result in entry.content: break
elif i == p.content.high: return
result.contentType = "multipart/form-data; boundary=" & bound
result.body = ""
for s in p.content:
result.body.add("--" & bound & "\c\L" & s)
result.body.add("--" & bound & "--\c\L")
proc sendFile(socket: Socket | AsyncSocket,
entry: MultipartEntry) {.multisync.} =
const chunkSize = 2^18
let file =
when socket is AsyncSocket: openAsync(entry.content)
else: newFileStream(entry.content, fmRead)
var buffer: string
while true:
buffer =
when socket is AsyncSocket: (await read(file, chunkSize)).string
else: readStr(file, chunkSize).string
if buffer.len == 0: break
await socket.send(buffer)
file.close()
proc redirection(status: string): bool =
const redirectionNRs = ["301", "302", "303", "307"]
@@ -427,8 +462,8 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string =
parsed.anchor = r.anchor
result = $parsed
proc generateHeaders(requestUrl: Uri, httpMethod: string,
headers: HttpHeaders, body: string, proxy: Proxy): string =
proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders,
proxy: Proxy): string =
# GET
let upperMethod = httpMethod.toUpperAscii()
result = upperMethod
@@ -447,34 +482,28 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string,
result.add($modifiedUrl)
# HTTP/1.1\c\l
result.add(" HTTP/1.1\c\L")
result.add(" HTTP/1.1" & httpNewLine)
# Host header.
if not headers.hasKey("Host"):
if requestUrl.port == "":
add(result, "Host: " & requestUrl.hostname & "\c\L")
add(result, "Host: " & requestUrl.hostname & httpNewLine)
else:
add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & "\c\L")
add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & httpNewLine)
# Connection header.
if not headers.hasKey("Connection"):
add(result, "Connection: Keep-Alive\c\L")
# Content length header.
const requiresBody = ["POST", "PUT", "PATCH"]
let needsContentLength = body.len > 0 or upperMethod in requiresBody
if needsContentLength and not headers.hasKey("Content-Length"):
add(result, "Content-Length: " & $body.len & "\c\L")
add(result, "Connection: Keep-Alive" & httpNewLine)
# Proxy auth header.
if not proxy.isNil and proxy.auth != "":
let auth = base64.encode(proxy.auth)
add(result, "Proxy-Authorization: basic " & auth & "\c\L")
add(result, "Proxy-Authorization: basic " & auth & httpNewLine)
for key, val in headers:
add(result, key & ": " & val & "\c\L")
add(result, key & ": " & val & httpNewLine)
add(result, "\c\L")
add(result, httpNewLine)
type
ProgressChangedProc*[ReturnType] =
@@ -511,9 +540,9 @@ type
type
HttpClient* = HttpClientBase[Socket]
proc newHttpClient*(userAgent = defUserAgent,
maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil,
timeout = -1, headers = newHttpHeaders()): HttpClient =
proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
sslContext = getDefaultSSL(), proxy: Proxy = nil,
timeout = -1, headers = newHttpHeaders()): HttpClient =
## Creates a new HttpClient instance.
##
## ``userAgent`` specifies the user agent that will be used when making
@@ -546,9 +575,9 @@ proc newHttpClient*(userAgent = defUserAgent,
type
AsyncHttpClient* = HttpClientBase[AsyncSocket]
proc newAsyncHttpClient*(userAgent = defUserAgent,
maxRedirects = 5, sslContext = getDefaultSSL(),
proxy: Proxy = nil, headers = newHttpHeaders()): AsyncHttpClient =
proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
sslContext = getDefaultSSL(), proxy: Proxy = nil,
headers = newHttpHeaders()): AsyncHttpClient =
## Creates a new AsyncHttpClient instance.
##
## ``userAgent`` specifies the user agent that will be used when making
@@ -671,8 +700,7 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void]
# Trailer headers will only be sent if the request specifies that we want
# them: http://tools.ietf.org/html/rfc2616#section-3.6.1
proc parseBody(client: HttpClient | AsyncHttpClient,
headers: HttpHeaders,
proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders,
httpVersion: string): Future[void] {.multisync.} =
# Reset progress from previous requests.
client.contentTotal = 0
@@ -745,7 +773,7 @@ proc parseResponse(client: HttpClient | AsyncHttpClient,
# We've been disconnected.
client.close()
break
if line == "\c\L":
if line == httpNewLine:
fullyRead = true
break
if not parsedStatus:
@@ -846,7 +874,7 @@ proc newConnection(client: HttpClient | AsyncHttpClient,
connectUrl.port = if url.port != "": url.port else: "443"
let proxyHeaderString = generateHeaders(connectUrl, $HttpConnect,
newHttpHeaders(), "", client.proxy)
newHttpHeaders(), client.proxy)
await client.socket.send(proxyHeaderString)
let proxyResp = await parseResponse(client, false)
@@ -864,6 +892,45 @@ proc newConnection(client: HttpClient | AsyncHttpClient,
client.currentURL = url
client.connected = true
proc readFileSizes(client: HttpClient | AsyncHttpClient,
multipart: MultipartData) {.multisync.} =
for entry in multipart.content.mitems():
if not entry.isFile: continue
if not entry.isStream:
entry.fileSize = entry.content.len
continue
# TODO: look into making getFileSize work with async
let fileSize = getFileSize(entry.content)
entry.fileSize = fileSize
proc format(entry: MultipartEntry, boundary: string): string =
result = "--" & boundary & httpNewLine
result.add("Content-Disposition: form-data; name=\"" & entry.name & "\"")
if entry.isFile:
result.add("; filename=\"" & entry.filename & "\"" & httpNewLine)
result.add("Content-Type: " & entry.contentType & httpNewLine)
else:
result.add(httpNewLine & httpNewLine & entry.content)
proc format(client: HttpClient | AsyncHttpClient,
multipart: MultipartData): Future[seq[string]] {.multisync.} =
let bound = getBoundary(multipart)
client.headers["Content-Type"] = "multipart/form-data; boundary=" & bound
await client.readFileSizes(multipart)
var length: int64
for entry in multipart.content:
result.add(format(entry, bound) & httpNewLine)
if entry.isFile:
length += entry.fileSize + httpNewLine.len
result.add "--" & bound & "--"
for s in result: length += s.len
client.headers["Content-Length"] = $length
proc override(fallback, override: HttpHeaders): HttpHeaders =
# Right-biased map union for `HttpHeaders`
if override.isNil:
@@ -875,9 +942,9 @@ proc override(fallback, override: HttpHeaders): HttpHeaders =
for k, vs in override.table:
result[k] = vs
proc requestAux(client: HttpClient | AsyncHttpClient, url: string,
httpMethod: string, body = "",
headers: HttpHeaders = nil): Future[Response | AsyncResponse]
proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string,
body = "", headers: HttpHeaders = nil,
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
# Helper that actually makes the request. Does not handle redirects.
let requestUrl = parseUri(url)
@@ -885,6 +952,12 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string,
if requestUrl.scheme == "":
raise newException(ValueError, "No uri scheme supplied.")
var data: seq[string]
if multipart != nil and multipart.content.len > 0:
data = await client.format(multipart)
else:
client.headers["Content-Length"] = $body.len
when client is AsyncHttpClient:
if not client.parseBodyFut.isNil:
# let the current operation finish before making another request
@@ -893,16 +966,30 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string,
await newConnection(client, requestUrl)
let effectiveHeaders = client.headers.override(headers)
let newHeaders = client.headers.override(headers)
if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0:
newHeaders["User-Agent"] = client.userAgent
if not effectiveHeaders.hasKey("user-agent") and client.userAgent != "":
effectiveHeaders["User-Agent"] = client.userAgent
let headerString = generateHeaders(requestUrl, httpMethod, newHeaders,
client.proxy)
await client.socket.send(headerString)
var headersString = generateHeaders(requestUrl, httpMethod,
effectiveHeaders, body, client.proxy)
await client.socket.send(headersString)
if body != "":
if data.len > 0:
var buffer: string
for i, entry in multipart.content:
buffer.add data[i]
if not entry.isFile: continue
if buffer.len > 0:
await client.socket.send(buffer)
buffer.setLen(0)
if entry.isStream:
await client.socket.sendFile(entry)
else:
await client.socket.send(entry.content)
buffer.add httpNewLine
# send the rest and the last boundary
await client.socket.send(buffer & data[^1])
elif body.len > 0:
await client.socket.send(body)
let getBody = httpMethod.toLowerAscii() notin ["head", "connect"] and
@@ -910,8 +997,8 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string,
result = await parseResponse(client, getBody)
proc request*(client: HttpClient | AsyncHttpClient, url: string,
httpMethod: string, body = "",
headers: HttpHeaders = nil): Future[Response | AsyncResponse]
httpMethod: string, body = "", headers: HttpHeaders = nil,
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
## Connects to the hostname specified by the URL and performs a request
## using the custom method string specified by ``httpMethod``.
@@ -922,7 +1009,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
##
## This procedure will follow redirects up to a maximum number of redirects
## specified in ``client.maxRedirects``.
result = await client.requestAux(url, httpMethod, body, headers)
result = await client.requestAux(url, httpMethod, body, headers, multipart)
var lastURL = url
for i in 1..client.maxRedirects:
@@ -930,13 +1017,12 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
let redirectTo = getNewLocation(lastURL, result.headers)
# Guarantee method for HTTP 307: see https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307
var meth = if result.status == "307": httpMethod else: "GET"
result = await client.requestAux(redirectTo, meth, body, headers)
result = await client.requestAux(redirectTo, meth, body, headers, multipart)
lastURL = redirectTo
proc request*(client: HttpClient | AsyncHttpClient, url: string,
httpMethod = HttpGet, body = "",
headers: HttpHeaders = nil): Future[Response | AsyncResponse]
httpMethod = HttpGet, body = "", headers: HttpHeaders = nil,
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
## Connects to the hostname specified by the URL and performs a request
## using the method specified.
@@ -947,7 +1033,7 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
##
## When a request is made to a different hostname, the current connection will
## be closed.
result = await request(client, url, $httpMethod, body, headers)
result = await request(client, url, $httpMethod, body, headers, multipart)
proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} =
## Returns the content of a response as a string.
@@ -980,42 +1066,25 @@ proc getContent*(client: HttpClient | AsyncHttpClient,
return await responseContent(resp)
proc delete*(client: HttpClient | AsyncHttpClient,
url: string): Future[Response | AsyncResponse] {.multisync.} =
url: string): Future[Response | AsyncResponse] {.multisync.} =
## Connects to the hostname specified by the URL and performs a DELETE request.
## This procedure uses httpClient values such as ``client.maxRedirects``.
result = await client.request(url, HttpDelete)
proc deleteContent*(client: HttpClient | AsyncHttpClient,
url: string): Future[string] {.multisync.} =
url: string): Future[string] {.multisync.} =
## Connects to the hostname specified by the URL and returns the content of a DELETE request.
let resp = await delete(client, url)
return await responseContent(resp)
proc makeRequestContent(body = "", multipart: MultipartData = nil): (string, HttpHeaders) =
let (mpContentType, mpBody) = format(multipart)
# TODO: Support FutureStream for `body` parameter.
template withNewLine(x): untyped =
if x.len > 0 and not x.endsWith("\c\L"):
x & "\c\L"
else:
x
var xb = mpBody.withNewLine() & body
var headers = newHttpHeaders()
if multipart != nil:
headers["Content-Type"] = mpContentType
headers["Content-Length"] = $len(xb)
return (xb, headers)
proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "",
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
## Connects to the hostname specified by the URL and performs a POST request.
## This procedure uses httpClient values such as ``client.maxRedirects``.
var (xb, headers) = makeRequestContent(body, multipart)
result = await client.request(url, $HttpPost, xb, headers)
result = await client.request(url, $HttpPost, body, multipart=multipart)
proc postContent*(client: HttpClient | AsyncHttpClient, url: string,
body = "",
proc postContent*(client: HttpClient | AsyncHttpClient, url: string, body = "",
multipart: MultipartData = nil): Future[string]
{.multisync.} =
## Connects to the hostname specified by the URL and returns the content of a POST request.
@@ -1023,32 +1092,27 @@ proc postContent*(client: HttpClient | AsyncHttpClient, url: string,
return await responseContent(resp)
proc put*(client: HttpClient | AsyncHttpClient, url: string, body = "",
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
## Connects to the hostname specified by the URL and performs a PUT request.
## This procedure uses httpClient values such as ``client.maxRedirects``.
var (xb, headers) = makeRequestContent(body, multipart)
result = await client.request(url, $HttpPut, xb, headers)
result = await client.request(url, $HttpPut, body, multipart=multipart)
proc putContent*(client: HttpClient | AsyncHttpClient, url: string,
body = "",
multipart: MultipartData = nil): Future[string]
{.multisync.} =
proc putContent*(client: HttpClient | AsyncHttpClient, url: string, body = "",
multipart: MultipartData = nil): Future[string] {.multisync.} =
## Connects to the hostname specified by the URL andreturns the content of a PUT request.
let resp = await put(client, url, body, multipart)
return await responseContent(resp)
proc patch*(client: HttpClient | AsyncHttpClient, url: string, body = "",
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
multipart: MultipartData = nil): Future[Response | AsyncResponse]
{.multisync.} =
## Connects to the hostname specified by the URL and performs a PATCH request.
## This procedure uses httpClient values such as ``client.maxRedirects``.
var (xb, headers) = makeRequestContent(body, multipart)
result = await client.request(url, $HttpPatch, xb, headers)
result = await client.request(url, $HttpPatch, body, multipart=multipart)
proc patchContent*(client: HttpClient | AsyncHttpClient, url: string,
body = "",
multipart: MultipartData = nil): Future[string]
proc patchContent*(client: HttpClient | AsyncHttpClient, url: string, body = "",
multipart: MultipartData = nil): Future[string]
{.multisync.} =
## Connects to the hostname specified by the URL and returns the content of a PATCH request.
let resp = await patch(client, url, body, multipart)

View File

@@ -97,6 +97,7 @@ const
Http504* = HttpCode(504)
Http505* = HttpCode(505)
const httpNewLine* = "\c\L"
const headerLimit* = 10_000
proc newHttpHeaders*(): HttpHeaders =