diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 557ddf8816..b92b56bdf3 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -220,7 +220,16 @@ ## ```Nim ## import std/httpclient ## -## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let myProxy = newProxy("http://user:password@myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## SOCKS5 proxy with proxy-side DNS resolving: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("socks5h://user:password@myproxy.network") ## let client = newHttpClient(proxy = myProxy) ## ``` ## @@ -338,7 +347,6 @@ proc body*(response: AsyncResponse): Future[string] {.async.} = type Proxy* = ref object url*: Uri - auth*: string MultipartEntry = object name, content: string @@ -387,13 +395,30 @@ proc getDefaultSSL(): SslContext = result = defaultSslContext doAssert result != nil, "failure to initialize the SSL context" -proc newProxy*(url: string; auth = ""): Proxy = +proc newProxy*(url: Uri): Proxy = ## Constructs a new `TProxy` object. - result = Proxy(url: parseUri(url), auth: auth) + result = Proxy(url: url) -proc newProxy*(url: Uri; auth = ""): Proxy = +proc newProxy*(url: string): Proxy = ## Constructs a new `TProxy` object. - result = Proxy(url: url, auth: auth) + result = Proxy(url: parseUri(url)) + +proc newProxy*(url: Uri; auth: string): Proxy {.deprecated: "Provide auth in url instead".} = + result = Proxy(url: url) + if auth != "": + let parts = auth.split(':') + if parts.len != 2: + raise newException(ValueError, "Invalid auth string") + result.url.username = parts[0] + result.url.password = parts[1] + +proc newProxy*(url: string; auth: string): Proxy {.deprecated: "Provide auth in url instead".} = + result = newProxy(parseUri(url), auth) + +proc auth*(p: Proxy): string {.deprecated: "Get auth from p.url.username and p.url.password".} = + result = "" + if p.url.username != "" or p.url.password != "": + result = p.url.username & ":" & p.url.password proc newMultipartData*: MultipartData {.inline.} = ## Constructs a new `MultipartData` object. @@ -548,7 +573,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade result = $httpMethod result.add ' ' - if proxy.isNil or requestUrl.scheme == "https": + if proxy.isNil or (requestUrl.scheme == "https" and proxy.url.scheme == "socks5h"): # /path?query if not requestUrl.path.startsWith("/"): result.add '/' result.add(requestUrl.path) @@ -575,8 +600,8 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade add(result, "Connection: Keep-Alive" & httpNewLine) # Proxy auth header. - if not proxy.isNil and proxy.auth != "": - let auth = base64.encode(proxy.auth) + if not proxy.isNil and proxy.url.username != "": + let auth = base64.encode(proxy.url.username & ":" & proxy.url.password) add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) for key, val in headers: @@ -689,7 +714,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, let exampleHtml = waitFor asyncProc() assert "Example Domain" in exampleHtml assert "Pizza" notin exampleHtml - + new result result.headers = headers result.userAgent = userAgent @@ -941,17 +966,75 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, when client is AsyncHttpClient: result.bodyStream.complete() +proc startSsl(client: HttpClient | AsyncHttpClient, hostname: string) = + when defined(ssl): + try: + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, hostname) + except: + client.socket.close() + raise getCurrentException() + +proc socks5hHandshake(client: HttpClient | AsyncHttpClient, + url: Uri) {.multisync.} = + var hasAuth = client.proxy.url.username != "" + if hasAuth: + await client.socket.send("\x05\x02\x00\x02") # Propose auth + else: + await client.socket.send("\x05\x01\x00") # Connect with no auth + + when client.socket is Socket: + var resp = client.socket.recv(2, client.timeout) + else: + var resp = await client.socket.recv(2) + + if resp == "\x05\x02" and hasAuth: + # Perform auth + let authStr = "\x01" & + char(client.proxy.url.username.len) & client.proxy.url.username & + char(client.proxy.url.password.len) & client.proxy.url.password + await client.socket.send(authStr) + when client.socket is Socket: + resp = client.socket.recv(2, client.timeout) + else: + resp = await client.socket.recv(2) + if resp != "\x01\x00": + httpError("Proxy authentication failed") + elif resp != "\x05\x00": + httpError("Unexpected proxy response: " & resp.toHex()) + + let port = if url.port != "": parseInt(url.port) + elif url.scheme == "http": 80 + else: 443 + var p = " " + p[0] = cast[char](port.uint16 shr 8) + p[1] = cast[char](port) + await client.socket.send("\x05\x01\x00\x03" & url.hostname.len.char & url.hostname & p) + when client.socket is Socket: + resp = client.socket.recv(10, client.timeout) + else: + resp = await client.socket.recv(10) + if resp.len != 10 or resp[0] != '\x05' or resp[1] != '\x00': + httpError("Unexpected proxy response: " & resp.toHex()) + proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = if client.currentURL.hostname != url.hostname or client.currentURL.scheme != url.scheme or client.currentURL.port != url.port or (not client.connected): - # Connect to proxy if specified - let connectionUrl = - if client.proxy.isNil: url else: client.proxy.url - let isSsl = connectionUrl.scheme.toLowerAscii() == "https" + var isSsl = false + var connectionUrl = url + if client.proxy.isNil: + isSsl = url.scheme.toLowerAscii() == "https" + else: + connectionUrl = client.proxy.url + let proxyScheme = connectionUrl.scheme.toLowerAscii() + if proxyScheme == "https": + isSsl = true + elif proxyScheme == "socks5h": + isSsl = url.scheme.toLowerAscii() == "https" if isSsl and not defined(ssl): raise newException(HttpRequestError, @@ -976,37 +1059,33 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.socket = await asyncnet.dial(connectionUrl.hostname, port) else: {.fatal: "Unsupported client type".} - when defined(ssl): - if isSsl: - try: + if not client.proxy.isNil and client.proxy.url.scheme.toLowerAscii() == "socks5h": + await socks5hHandshake(client, url) + if isSsl: startSsl(client, url.hostname) + else: + if isSsl: startSsl(client, connectionUrl.hostname) + # If need to CONNECT through http(s) proxy + if url.scheme == "https" and not client.proxy.isNil: + when defined(ssl): + # Pass only host:port for CONNECT + var connectUrl = initUri() + connectUrl.hostname = url.hostname + connectUrl.port = if url.port != "": url.port else: "443" + + let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, + newHttpHeaders(), client.proxy) + await client.socket.send(proxyHeaderString) + let proxyResp = await parseResponse(client, false) + + if not proxyResp.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") client.sslContext.wrapConnectedSocket( - client.socket, handshakeAsClient, connectionUrl.hostname) - except: - client.socket.close() - raise getCurrentException() - - # If need to CONNECT through proxy - if url.scheme == "https" and not client.proxy.isNil: - when defined(ssl): - # Pass only host:port for CONNECT - var connectUrl = initUri() - connectUrl.hostname = url.hostname - connectUrl.port = if url.port != "": url.port else: "443" - - let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, - newHttpHeaders(), client.proxy) - await client.socket.send(proxyHeaderString) - let proxyResp = await parseResponse(client, false) - - if not proxyResp.status.startsWith("200"): + client.socket, handshakeAsClient, url.hostname) + else: raise newException(HttpRequestError, - "The proxy server rejected a CONNECT request, " & - "so a secure connection could not be established.") - client.sslContext.wrapConnectedSocket( - client.socket, handshakeAsClient, url.hostname) - else: - raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") # May be connected through proxy but remember actual URL being accessed client.currentURL = url @@ -1086,7 +1165,7 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, var data: seq[string] = @[] if multipart != nil and multipart.content.len > 0: - # `format` modifies `client.headers`, see + # `format` modifies `client.headers`, see # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 data = await client.format(multipart) newHeaders = client.headers.override(headers) @@ -1319,7 +1398,7 @@ proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) = defer: client.getBody = true let resp = client.get(url) - + if resp.code.is4xx or resp.code.is5xx: raise newException(HttpRequestError, resp.status) @@ -1334,7 +1413,7 @@ proc downloadFileEx(client: AsyncHttpClient, ## Downloads `url` and saves it to `filename`. client.getBody = false let resp = await client.get(url) - + if resp.code.is4xx or resp.code.is5xx: raise newException(HttpRequestError, resp.status) diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim index 0bd4796704..99ccaba8b3 100644 --- a/tests/stdlib/thttpclient.nim +++ b/tests/stdlib/thttpclient.nim @@ -107,6 +107,13 @@ proc asyncTest() {.async.} = # client = newAsyncHttpClient(proxy = newProxy("http://51.254.106.76:80/")) # var resp = await client.request("https://github.com") # echo resp + # + # SOCKS5H proxy test + # when manualTests: + # block: + # client = newAsyncHttpClient(proxy = newProxy("socks5h://user:blabla@127.0.0.1:9050")) + # var resp = await client.request("https://api.my-ip.io/v2/ip.txt") + # echo await resp.body proc syncTest() = var client = newHttpClient()