mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2025-12-29 01:14:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

SOCKS5H support for httpclient (#25070)

Browse Source 
- Added support for SOCKS5h (h for proxy-side DNS resolving) to
httpclient
- Deprecated `auth` arguments for `newProxy` constructors, for auth to
be embedded in the url.

Unfortunately `http://example.com` is not currently reachable from
github CI, so the tests fail there for a few days already, I'm not sure
what can be done here.

(cherry picked from commit 161b321796)
This commit is contained in:
Yuriy Glukhov
2025-07-30 00:14:41 +02:00
committed by narimiran
parent 23b7372aa0
commit ca74debfbf
2 changed files with 132 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
lib/pure/httpclient.nim
View File

@@ -220,7 +220,16 @@
## ```Nim
## import std/httpclient
##
## let myProxy = newProxy("http://myproxy.network", auth="user:password")
## let myProxy = newProxy("http://user:password@myproxy.network")
## let client = newHttpClient(proxy = myProxy)
## ```
##
## SOCKS5 proxy with proxy-side DNS resolving:
##
## ```Nim
## import std/httpclient
##
## let myProxy = newProxy("socks5h://user:password@myproxy.network")
## let client = newHttpClient(proxy = myProxy)
## ```
##
@@ -338,7 +347,6 @@ proc body*(response: AsyncResponse): Future[string] {.async.} =
type
Proxy* = ref object
url*: Uri
auth*: string
MultipartEntry = object
name, content: string
@@ -387,13 +395,30 @@ proc getDefaultSSL(): SslContext =
result = defaultSslContext
doAssert result != nil, "failure to initialize the SSL context"
proc newProxy*(url: string; auth = ""): Proxy =
proc newProxy*(url: Uri): Proxy =
## Constructs a new `TProxy` object.
result = Proxy(url: parseUri(url), auth: auth)
result = Proxy(url: url)
proc newProxy*(url: Uri; auth = ""): Proxy =
proc newProxy*(url: string): Proxy =
## Constructs a new `TProxy` object.
result = Proxy(url: url, auth: auth)
result = Proxy(url: parseUri(url))
proc newProxy*(url: Uri; auth: string): Proxy {.deprecated: "Provide auth in url instead".} =
result = Proxy(url: url)
if auth != "":
let parts = auth.split(':')
if parts.len != 2:
raise newException(ValueError, "Invalid auth string")
result.url.username = parts[0]
result.url.password = parts[1]
proc newProxy*(url: string; auth: string): Proxy {.deprecated: "Provide auth in url instead".} =
result = newProxy(parseUri(url), auth)
proc auth*(p: Proxy): string {.deprecated: "Get auth from p.url.username and p.url.password".} =
result = ""
if p.url.username != "" or p.url.password != "":
result = p.url.username & ":" & p.url.password
proc newMultipartData*: MultipartData {.inline.} =
## Constructs a new `MultipartData` object.
@@ -548,7 +573,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade
result = $httpMethod
result.add ' '
if proxy.isNil or requestUrl.scheme == "https":
if proxy.isNil or (requestUrl.scheme == "https" and proxy.url.scheme == "socks5h"):
# /path?query
if not requestUrl.path.startsWith("/"): result.add '/'
result.add(requestUrl.path)
@@ -575,8 +600,8 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade
add(result, "Connection: Keep-Alive" & httpNewLine)
# Proxy auth header.
if not proxy.isNil and proxy.auth != "":
let auth = base64.encode(proxy.auth)
if not proxy.isNil and proxy.url.username != "":
let auth = base64.encode(proxy.url.username & ":" & proxy.url.password)
add(result, "Proxy-Authorization: Basic " & auth & httpNewLine)
for key, val in headers:
@@ -689,7 +714,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
let exampleHtml = waitFor asyncProc()
assert "Example Domain" in exampleHtml
assert "Pizza" notin exampleHtml
new result
result.headers = headers
result.userAgent = userAgent
@@ -941,17 +966,75 @@ proc parseResponse(client: HttpClient | AsyncHttpClient,
when client is AsyncHttpClient:
result.bodyStream.complete()
proc startSsl(client: HttpClient | AsyncHttpClient, hostname: string) =
when defined(ssl):
try:
client.sslContext.wrapConnectedSocket(
client.socket, handshakeAsClient, hostname)
except:
client.socket.close()
raise getCurrentException()
proc socks5hHandshake(client: HttpClient | AsyncHttpClient,
url: Uri) {.multisync.} =
var hasAuth = client.proxy.url.username != ""
if hasAuth:
await client.socket.send("\x05\x02\x00\x02") # Propose auth
else:
await client.socket.send("\x05\x01\x00") # Connect with no auth
when client.socket is Socket:
var resp = client.socket.recv(2, client.timeout)
else:
var resp = await client.socket.recv(2)
if resp == "\x05\x02" and hasAuth:
# Perform auth
let authStr = "\x01" &
char(client.proxy.url.username.len) & client.proxy.url.username &
char(client.proxy.url.password.len) & client.proxy.url.password
await client.socket.send(authStr)
when client.socket is Socket:
resp = client.socket.recv(2, client.timeout)
else:
resp = await client.socket.recv(2)
if resp != "\x01\x00":
httpError("Proxy authentication failed")
elif resp != "\x05\x00":
httpError("Unexpected proxy response: " & resp.toHex())
let port = if url.port != "": parseInt(url.port)
elif url.scheme == "http": 80
else: 443
var p = " "
p[0] = cast[char](port.uint16 shr 8)
p[1] = cast[char](port)
await client.socket.send("\x05\x01\x00\x03" & url.hostname.len.char & url.hostname & p)
when client.socket is Socket:
resp = client.socket.recv(10, client.timeout)
else:
resp = await client.socket.recv(10)
if resp.len != 10 or resp[0] != '\x05' or resp[1] != '\x00':
httpError("Unexpected proxy response: " & resp.toHex())
proc newConnection(client: HttpClient | AsyncHttpClient,
url: Uri) {.multisync.} =
if client.currentURL.hostname != url.hostname or
client.currentURL.scheme != url.scheme or
client.currentURL.port != url.port or
(not client.connected):
# Connect to proxy if specified
let connectionUrl =
if client.proxy.isNil: url else: client.proxy.url
let isSsl = connectionUrl.scheme.toLowerAscii() == "https"
var isSsl = false
var connectionUrl = url
if client.proxy.isNil:
isSsl = url.scheme.toLowerAscii() == "https"
else:
connectionUrl = client.proxy.url
let proxyScheme = connectionUrl.scheme.toLowerAscii()
if proxyScheme == "https":
isSsl = true
elif proxyScheme == "socks5h":
isSsl = url.scheme.toLowerAscii() == "https"
if isSsl and not defined(ssl):
raise newException(HttpRequestError,
@@ -976,37 +1059,33 @@ proc newConnection(client: HttpClient | AsyncHttpClient,
client.socket = await asyncnet.dial(connectionUrl.hostname, port)
else: {.fatal: "Unsupported client type".}
when defined(ssl):
if isSsl:
try:
if not client.proxy.isNil and client.proxy.url.scheme.toLowerAscii() == "socks5h":
await socks5hHandshake(client, url)
if isSsl: startSsl(client, url.hostname)
else:
if isSsl: startSsl(client, connectionUrl.hostname)
# If need to CONNECT through http(s) proxy
if url.scheme == "https" and not client.proxy.isNil:
when defined(ssl):
# Pass only host:port for CONNECT
var connectUrl = initUri()
connectUrl.hostname = url.hostname
connectUrl.port = if url.port != "": url.port else: "443"
let proxyHeaderString = generateHeaders(connectUrl, HttpConnect,
newHttpHeaders(), client.proxy)
await client.socket.send(proxyHeaderString)
let proxyResp = await parseResponse(client, false)
if not proxyResp.status.startsWith("200"):
raise newException(HttpRequestError,
"The proxy server rejected a CONNECT request, " &
"so a secure connection could not be established.")
client.sslContext.wrapConnectedSocket(
client.socket, handshakeAsClient, connectionUrl.hostname)
except:
client.socket.close()
raise getCurrentException()
# If need to CONNECT through proxy
if url.scheme == "https" and not client.proxy.isNil:
when defined(ssl):
# Pass only host:port for CONNECT
var connectUrl = initUri()
connectUrl.hostname = url.hostname
connectUrl.port = if url.port != "": url.port else: "443"
let proxyHeaderString = generateHeaders(connectUrl, HttpConnect,
newHttpHeaders(), client.proxy)
await client.socket.send(proxyHeaderString)
let proxyResp = await parseResponse(client, false)
if not proxyResp.status.startsWith("200"):
client.socket, handshakeAsClient, url.hostname)
else:
raise newException(HttpRequestError,
"The proxy server rejected a CONNECT request, " &
"so a secure connection could not be established.")
client.sslContext.wrapConnectedSocket(
client.socket, handshakeAsClient, url.hostname)
else:
raise newException(HttpRequestError,
"SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.")
"SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.")
# May be connected through proxy but remember actual URL being accessed
client.currentURL = url
@@ -1086,7 +1165,7 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri,
var data: seq[string] = @[]
if multipart != nil and multipart.content.len > 0:
# `format` modifies `client.headers`, see
# `format` modifies `client.headers`, see
# https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979
data = await client.format(multipart)
newHeaders = client.headers.override(headers)
@@ -1319,7 +1398,7 @@ proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) =
defer:
client.getBody = true
let resp = client.get(url)
if resp.code.is4xx or resp.code.is5xx:
raise newException(HttpRequestError, resp.status)
@@ -1334,7 +1413,7 @@ proc downloadFileEx(client: AsyncHttpClient,
## Downloads `url` and saves it to `filename`.
client.getBody = false
let resp = await client.get(url)
if resp.code.is4xx or resp.code.is5xx:
raise newException(HttpRequestError, resp.status)

7
tests/stdlib/thttpclient.nim
View File

@@ -107,6 +107,13 @@ proc asyncTest() {.async.} =
# client = newAsyncHttpClient(proxy = newProxy("http://51.254.106.76:80/"))
# var resp = await client.request("https://github.com")
# echo resp
#
# SOCKS5H proxy test
# when manualTests:
# block:
# client = newAsyncHttpClient(proxy = newProxy("socks5h://user:blabla@127.0.0.1:9050"))
# var resp = await client.request("https://api.my-ip.io/v2/ip.txt")
# echo await resp.body
proc syncTest() =
var client = newHttpClient()