From 7a7fe85f85126527c029544a9573d2ca38d2bd77 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 17 Dec 2014 17:53:11 +0000 Subject: [PATCH 1/6] switched httpclient to use net module instead of sockets --- lib/pure/httpclient.nim | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 2b161778c0..dc800fa819 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -88,10 +88,9 @@ ## constructor should be used for this purpose. However, ## currently only basic authentication is supported. -import sockets, strutils, parseurl, parseutils, strtabs, base64, os +import net, strutils, parseurl, parseutils, strtabs, base64, os import asyncnet, asyncdispatch import rawsockets -from net import nil type Response* = tuple[ @@ -308,18 +307,18 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "", add(headers, extraHeaders) add(headers, "\c\L") - var s = socket() - if s == invalidSocket: raiseOSError(osLastError()) - var port = sockets.Port(80) + var s = newSocket() + if s == nil: raiseOSError(osLastError()) + var port = net.Port(80) if r.scheme == "https": when defined(ssl): sslContext.wrapSocket(s) - port = sockets.Port(443) + port = net.Port(443) else: raise newException(HttpRequestError, "SSL support is not available. Cannot connect over SSL.") if r.port != "": - port = sockets.Port(r.port.parseInt) + port = net.Port(r.port.parseInt) if timeout == -1: s.connect(r.hostname, port) From 3cfc8877f619b9034a2ddaa892a872d9efd993cc Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 17 Dec 2014 18:16:06 +0000 Subject: [PATCH 2/6] replace deprecated parseUrl with parseUri in httpclient --- lib/pure/httpclient.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index dc800fa819..f3f6f8da65 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -88,7 +88,7 @@ ## constructor should be used for this purpose. However, ## currently only basic authentication is supported. -import net, strutils, parseurl, parseutils, strtabs, base64, os +import net, strutils, uri, parseutils, strtabs, base64, os import asyncnet, asyncdispatch import rawsockets @@ -100,7 +100,7 @@ type body: string] Proxy* = ref object - url*: Url + url*: Uri auth*: string ProtocolError* = object of IOError ## exception that is raised when server @@ -278,7 +278,7 @@ else: proc newProxy*(url: string, auth = ""): Proxy = ## Constructs a new ``TProxy`` object. - result = Proxy(url: parseUrl(url), auth: auth) + result = Proxy(url: parseUri(url), auth: auth) proc request*(url: string, httpMethod = httpGET, extraHeaders = "", body = "", @@ -289,7 +289,7 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "", ## | Extra headers can be specified and must be seperated by ``\c\L`` ## | An optional timeout can be specified in miliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. - var r = if proxy == nil: parseUrl(url) else: proxy.url + var r = if proxy == nil: parseUri(url) else: proxy.url var headers = substr($httpMethod, len("http")) if proxy == nil: headers.add(" /" & r.path & r.query) @@ -341,9 +341,9 @@ proc getNewLocation(lastUrl: string, headers: StringTableRef): string = result = headers["Location"] if result == "": httpError("location header expected") # Relative URLs. (Not part of the spec, but soon will be.) - let r = parseURL(result) + let r = parseUri(result) if r.hostname == "" and r.path != "": - let origParsed = parseURL(lastUrl) + let origParsed = parseUri(lastUrl) result = origParsed.hostname & "/" & r.path proc get*(url: string, extraHeaders = "", maxRedirects = 5, @@ -436,7 +436,7 @@ proc downloadFile*(url: string, outputFilename: string, else: fileError("Unable to open file") -proc generateHeaders(r: Url, httpMethod: HttpMethod, +proc generateHeaders(r: Uri, httpMethod: HttpMethod, headers: StringTableRef): string = result = substr($httpMethod, len("http")) # TODO: Proxies @@ -454,7 +454,7 @@ type AsyncHttpClient* = ref object socket: AsyncSocket connected: bool - currentURL: Url ## Where we are currently connected. + currentURL: Uri ## Where we are currently connected. headers*: StringTableRef maxRedirects: int userAgent: string @@ -606,7 +606,7 @@ proc parseResponse(client: AsyncHttpClient, else: result.body = "" -proc newConnection(client: AsyncHttpClient, url: Url) {.async.} = +proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} = if client.currentURL.hostname != url.hostname or client.currentURL.scheme != url.scheme: if client.connected: client.close() @@ -642,7 +642,7 @@ proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET, ## connection can be closed by using the ``close`` procedure. ## ## The returned future will complete once the request is completed. - let r = parseUrl(url) + let r = parseUri(url) await newConnection(client, r) if not client.headers.hasKey("user-agent") and client.userAgent != "": From 2b854849ca84dc2238d6d886c330d1f4ae0450e2 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 17 Dec 2014 18:51:37 +0000 Subject: [PATCH 3/6] removed unused variable from httpclient --- lib/pure/httpclient.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index f3f6f8da65..a7e1aeab21 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -498,7 +498,6 @@ proc recvFull(socket: AsyncSocket, size: int): Future[string] {.async.} = proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = result = "" - var ri = 0 while true: var chunkSize = 0 var chunkSizeStr = await client.socket.recvLine() From d0ea2bdf5b6b28f5334400aca399e5bb7b2ce50e Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 17 Dec 2014 23:08:46 +0000 Subject: [PATCH 4/6] improved error msgs and refactored close in the net module --- lib/pure/net.nim | 40 ++++++++++++++++++++++++++++++++-------- lib/wrappers/openssl.nim | 13 ++++++++++--- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 8afc6c5c54..30d79ad9a2 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -256,7 +256,21 @@ proc socketError*(socket: Socket, err: int = -1, async = false, else: raiseSSLError("Not enough data on socket.") of SSL_ERROR_WANT_X509_LOOKUP: raiseSSLError("Function for x509 lookup has been called.") - of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: + of SSL_ERROR_SYSCALL: + var errStr = "IO error has occured " + let sslErr = ErrPeekLastError() + if sslErr == 0 and err == 0: + errStr.add "because an EOF was observed that violates the protocol" + elif sslErr == 0 and err == -1: + errStr.add "in the BIO layer" + else: + let errStr = $ErrErrorString(sslErr, nil) + raiseSSLError(errStr & ": " & errStr) + let osMsg = osErrorMsg osLastError() + if osMsg != "": + errStr.add ". The OS reports: " & osMsg + raise newException(OSError, errStr) + of SSL_ERROR_SSL: raiseSSLError() else: raiseSSLError("Unknown Error") @@ -416,17 +430,27 @@ proc accept*(server: Socket, client: var Socket, var addrDummy = "" acceptAddr(server, client, addrDummy, flags) -proc close*(socket: Socket) = +proc close*(socket: Socket, twoWay = true) = ## Closes a socket. - socket.fd.close() + ## + ## When used with a **SSL** socket, `twoWay` can be set to mandate a two-way + ## cryptographically secure shutdown, otherwise, a "close notify" shutdown + ## alert is sent and the underlying socket is closed without waiting for a response. + ## This is acceptable under the TLS standard when the underlying connection is not + ## going to be use for further communications. when defined(ssl): if socket.isSSL: - let res = SSLShutdown(socket.sslHandle) - if res == 0: - if SSLShutdown(socket.sslHandle) != 1: - socketError(socket) + ErrClearError() + var res = SSLShutdown(socket.sslHandle) + if res == 0 and twoWay: + res = SSLShutdown(socket.sslHandle) + if res != 1: + socketError(socket, res) + elif res == 0: + discard elif res != 1: - socketError(socket) + socketError(socket, res) + socket.fd.close() proc toCInt*(opt: SOBool): cint = ## Converts a ``SOBool`` into its Socket Option cint representation. diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 6e85fb9dd8..ba25fbf1a3 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -89,6 +89,8 @@ type {.deprecated: [PSSL: SslPtr, PSSL_CTX: SslCtx, PBIO: BIO].} const + SSL_SENT_SHUTDOWN* = 1 + SSL_RECEIVED_SHUTDOWN* = 2 EVP_MAX_MD_SIZE* = 16 + 20 SSL_ERROR_NONE* = 0 SSL_ERROR_SSL* = 1 @@ -233,6 +235,8 @@ proc SSL_CTX_check_private_key*(ctx: SslCtx): cInt{.cdecl, dynlib: DLLSSLName, proc SSL_set_fd*(ssl: SslPtr, fd: SocketHandle): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_shutdown*(ssl: SslPtr): cInt{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_set_shutdown*(ssl: SslPtr, mode: cint) {.cdecl, dynlib: DLLSSLName, importc: "SSL_set_shutdown".} +proc SSL_get_shutdown*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc: "SSL_get_shutdown".} proc SSL_connect*(ssl: SslPtr): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_read*(ssl: SslPtr, buf: pointer, num: int): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_write*(ssl: SslPtr, buf: cstring, num: int): cint{.cdecl, dynlib: DLLSSLName, importc.} @@ -314,6 +318,11 @@ proc sslDoHandshake*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc: "SSL_do_handshake".} + +proc ErrClearError*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_clear_error".} +proc ErrFreeStrings*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_free_strings".} +proc ErrRemoveState*(pid: cInt){.cdecl, dynlib: DLLUtilName, importc: "ERR_remove_state".} + when true: discard else: @@ -414,9 +423,7 @@ else: # function ErrErrorString(e: cInt; buf: PChar): PChar; proc SSLeayversion*(t: cInt): cstring{.cdecl, dynlib: DLLUtilName, importc.} - proc ErrClearError*(){.cdecl, dynlib: DLLUtilName, importc.} - proc ErrFreeStrings*(){.cdecl, dynlib: DLLUtilName, importc.} - proc ErrRemoveState*(pid: cInt){.cdecl, dynlib: DLLUtilName, importc.} + proc OPENSSLaddallalgorithms*(){.cdecl, dynlib: DLLUtilName, importc.} proc CRYPTOcleanupAllExData*(){.cdecl, dynlib: DLLUtilName, importc.} proc RandScreen*(){.cdecl, dynlib: DLLUtilName, importc.} From ff24ca99193190615252aa3d8ec2a56f9aaa3def Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 17 Dec 2014 23:27:11 +0000 Subject: [PATCH 5/6] realised extra parameter was unnecessary --- lib/pure/net.nim | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 30d79ad9a2..2b7b96c652 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -430,23 +430,17 @@ proc accept*(server: Socket, client: var Socket, var addrDummy = "" acceptAddr(server, client, addrDummy, flags) -proc close*(socket: Socket, twoWay = true) = +proc close*(socket: Socket) = ## Closes a socket. - ## - ## When used with a **SSL** socket, `twoWay` can be set to mandate a two-way - ## cryptographically secure shutdown, otherwise, a "close notify" shutdown - ## alert is sent and the underlying socket is closed without waiting for a response. - ## This is acceptable under the TLS standard when the underlying connection is not - ## going to be use for further communications. when defined(ssl): if socket.isSSL: ErrClearError() - var res = SSLShutdown(socket.sslHandle) - if res == 0 and twoWay: - res = SSLShutdown(socket.sslHandle) - if res != 1: - socketError(socket, res) - elif res == 0: + # As we are closing the underlying socket immediately afterwards, + # it is valid, under the TLS standard, to perform a unidirectional + # shutdown i.e not wait for the peers "close notify" alert with a second + # call to SSLShutdown + let res = SSLShutdown(socket.sslHandle) + if res == 0: discard elif res != 1: socketError(socket, res) From 1b614ffc8044d9efcf5ae5c9ad59f2416e8ccfa2 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 18 Dec 2014 00:45:02 +0000 Subject: [PATCH 6/6] ensure file descriptor is closed --- lib/pure/net.nim | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 2b7b96c652..28b84eb395 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -432,19 +432,21 @@ proc accept*(server: Socket, client: var Socket, proc close*(socket: Socket) = ## Closes a socket. - when defined(ssl): - if socket.isSSL: - ErrClearError() - # As we are closing the underlying socket immediately afterwards, - # it is valid, under the TLS standard, to perform a unidirectional - # shutdown i.e not wait for the peers "close notify" alert with a second - # call to SSLShutdown - let res = SSLShutdown(socket.sslHandle) - if res == 0: - discard - elif res != 1: - socketError(socket, res) - socket.fd.close() + try: + when defined(ssl): + if socket.isSSL: + ErrClearError() + # As we are closing the underlying socket immediately afterwards, + # it is valid, under the TLS standard, to perform a unidirectional + # shutdown i.e not wait for the peers "close notify" alert with a second + # call to SSLShutdown + let res = SSLShutdown(socket.sslHandle) + if res == 0: + discard + elif res != 1: + socketError(socket, res) + finally: + socket.fd.close() proc toCInt*(opt: SOBool): cint = ## Converts a ``SOBool`` into its Socket Option cint representation.