From 159643824b4307597ece03597a44579c5fbcc9ed Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 27 Mar 2017 09:39:46 +0200 Subject: [PATCH 1/8] fixes #3847 (#5609) --- lib/pure/asynchttpserver.nim | 34 +++++++++++++++++++++++++++------- lib/pure/asyncnet.nim | 16 ++++------------ lib/pure/httpcore.nim | 4 ++-- lib/pure/net.nim | 11 +++-------- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 80aa8bb2ce..8d059dbbc6 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -36,6 +36,9 @@ import httpcore export httpcore except parseHeader +const + maxLine = 8*1024 + # TODO: If it turns out that the decisions that asynchttpserver makes # explicitly, about whether to close the client sockets or upgrade them are # wrong, then add a return value which determines what to do for the callback. @@ -97,6 +100,18 @@ proc respond*(req: Request, code: HttpCode, content: string, if headers != nil: msg.addHeaders(headers) + msg.add("Content-Length: ") + # this particular way saves allocations: + msg.add content.len + msg.add "\c\L\c\L" + msg.add(content) + result = req.client.send(msg) + +proc respondError(req: Request, code: HttpCode): Future[void] = + ## Responds to the request with the specified ``HttpCode``. + let content = $code + var msg = "HTTP/1.1 " & content & "\c\L" + msg.add("Content-Length: " & $content.len & "\c\L\c\L") msg.add(content) result = req.client.send(msg) @@ -139,12 +154,16 @@ proc processClient(client: AsyncSocket, address: string, for i in 0..1: lineFut.mget().setLen(0) lineFut.clean() - await client.recvLineInto(lineFut) # TODO: Timeouts. + await client.recvLineInto(lineFut, maxLength=maxLine) # TODO: Timeouts. if lineFut.mget == "": client.close() return + if lineFut.mget.len > maxLine: + await request.respondError(Http413) + client.close() + return if lineFut.mget != "\c\L": break @@ -157,19 +176,17 @@ proc processClient(client: AsyncSocket, address: string, # TODO: this is likely slow. request.reqMethod = parseEnum[HttpMethod]("http" & linePart) except ValueError: - asyncCheck request.respond(Http400, "Invalid request method. Got: " & - linePart) + asyncCheck request.respondError(Http400) continue of 1: parseUri(linePart, request.url) of 2: try: request.protocol = parseProtocol(linePart) except ValueError: - asyncCheck request.respond(Http400, - "Invalid request protocol. Got: " & linePart) + asyncCheck request.respondError(Http400) continue else: - await request.respond(Http400, "Invalid request. Got: " & lineFut.mget) + await request.respondError(Http400) continue inc i @@ -178,10 +195,13 @@ proc processClient(client: AsyncSocket, address: string, i = 0 lineFut.mget.setLen(0) lineFut.clean() - await client.recvLineInto(lineFut) + await client.recvLineInto(lineFut, maxLength=maxLine) if lineFut.mget == "": client.close(); return + if lineFut.mget.len > maxLine: + await request.respondError(Http413) + client.close(); return if lineFut.mget == "\c\L": break let (key, value) = parseHeader(lineFut.mget) request.headers[key] = value diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 2d5c650012..1ec751a64a 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -464,8 +464,7 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], ## The partial line **will be lost**. ## ## The ``maxLength`` parameter determines the maximum amount of characters - ## that can be read before a ``ValueError`` is raised. This prevents Denial - ## of Service (DOS) attacks. + ## that can be read. ``resString`` will be truncated after that. ## ## **Warning**: The ``Peek`` flag is not yet implemented. ## @@ -519,10 +518,7 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], socket.currPos.inc() # Verify that this isn't a DOS attack: #3847. - if resString.mget.len > maxLength: - let msg = "recvLine received more than the specified `maxLength` " & - "allowed." - raise newException(ValueError, msg) + if resString.mget.len > maxLength: break else: var c = "" while true: @@ -546,10 +542,7 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], resString.mget.add c # Verify that this isn't a DOS attack: #3847. - if resString.mget.len > maxLength: - let msg = "recvLine received more than the specified `maxLength` " & - "allowed." - raise newException(ValueError, msg) + if resString.mget.len > maxLength: break resString.complete() proc recvLine*(socket: AsyncSocket, @@ -569,8 +562,7 @@ proc recvLine*(socket: AsyncSocket, ## The partial line **will be lost**. ## ## The ``maxLength`` parameter determines the maximum amount of characters - ## that can be read before a ``ValueError`` is raised. This prevents Denial - ## of Service (DOS) attacks. + ## that can be read. The result is truncated after that. ## ## **Warning**: The ``Peek`` flag is not yet implemented. ## diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index aa8f1958d1..a5ab40ca4b 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -285,7 +285,7 @@ proc `$`*(code: HttpCode): string = proc `==`*(a, b: HttpCode): bool {.borrow.} proc `==`*(rawCode: string, code: HttpCode): bool = - return rawCode.toLower() == ($code).toLower() + return cmpIgnoreCase(rawCode, $code) == 0 proc is2xx*(code: HttpCode): bool = ## Determines whether ``code`` is a 2xx HTTP status code. @@ -304,7 +304,7 @@ proc is5xx*(code: HttpCode): bool = return code.int in {500 .. 599} proc `$`*(httpMethod: HttpMethod): string = - return (system.`$`(httpMethod))[4 .. ^1].toUpper() + return (system.`$`(httpMethod))[4 .. ^1].toUpperAscii() when isMainModule: var test = newHttpHeaders() diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 7f67833582..56f8b93993 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1010,8 +1010,7 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, ## the specified time an ETimeout exception will be raised. ## ## The ``maxLength`` parameter determines the maximum amount of characters - ## that can be read before a ``ValueError`` is raised. This prevents Denial - ## of Service (DOS) attacks. + ## that can be read. The result is truncated after that. ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. @@ -1047,10 +1046,7 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, add(line.string, c) # Verify that this isn't a DOS attack: #3847. - if line.string.len > maxLength: - let msg = "recvLine received more than the specified `maxLength` " & - "allowed." - raise newException(ValueError, msg) + if line.string.len > maxLength: break proc recvLine*(socket: Socket, timeout = -1, flags = {SocketFlag.SafeDisconn}, @@ -1069,8 +1065,7 @@ proc recvLine*(socket: Socket, timeout = -1, ## the specified time an ETimeout exception will be raised. ## ## The ``maxLength`` parameter determines the maximum amount of characters - ## that can be read before a ``ValueError`` is raised. This prevents Denial - ## of Service (DOS) attacks. + ## that can be read. The result is truncated after that. ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. result = "" From e0bb65e45c8dec9683740f1a656f72f7eaa4d32f Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 27 Mar 2017 21:11:48 +0200 Subject: [PATCH 2/8] Fixes the order in which FutureVar and return completions are made. This caused a pretty bad and subtle bug in the asynchttpserver. As far as I can understand, the fact that the returned future was being completed first meant that the underlying async procedure could continue running and thus clean() the FutureVar and request new data. The control then went back and the FutureVar was completed again causing an error. --- lib/pure/asyncmacro.nim | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index f0837d67d5..ce4c9a9c94 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -40,6 +40,8 @@ template createCb(retFutureSym, iteratorNameSym, else: next.callback = cb except: + futureVarCompletions + if retFutureSym.finished: # Take a look at tasyncexceptions for the bug which this fixes. # That test explains it better than I can here. @@ -47,7 +49,6 @@ template createCb(retFutureSym, iteratorNameSym, else: retFutureSym.fail(getCurrentException()) - futureVarCompletions cb() #{.pop.} proc generateExceptionCheck(futSym, @@ -123,12 +124,16 @@ template createVar(result: var NimNode, futSymName: string, result.add newVarStmt(futSym, asyncProc) # -> var future = y useVar(result, futSym, valueReceiver, rootReceiver, fromNode) -proc createFutureVarCompletions(futureVarIdents: seq[NimNode]): NimNode - {.compileTime.} = - result = newStmtList() +proc createFutureVarCompletions(futureVarIdents: seq[NimNode], + fromNode: NimNode): NimNode {.compileTime.} = + result = newNimNode(nnkStmtList, fromNode) # Add calls to complete each FutureVar parameter. for ident in futureVarIdents: # Only complete them if they have not been completed already by the user. + # TODO: Once https://github.com/nim-lang/Nim/issues/5617 is fixed. + # TODO: Add line info to the complete() call! + # In the meantime, this was really useful for debugging :) + #result.add(newCall(newIdentNode("echo"), newStrLitNode(fromNode.lineinfo))) result.add newIfStmt( ( newCall(newIdentNode("not"), @@ -145,6 +150,10 @@ proc processBody(node, retFutureSym: NimNode, case node.kind of nnkReturnStmt: result = newNimNode(nnkStmtList, node) + + # As I've painfully found out, the order here really DOES matter. + result.add createFutureVarCompletions(futureVarIdents, node) + if node[0].kind == nnkEmpty: if not subTypeIsVoid: result.add newCall(newIdentNode("complete"), retFutureSym, @@ -158,8 +167,6 @@ proc processBody(node, retFutureSym: NimNode, else: result.add newCall(newIdentNode("complete"), retFutureSym, x) - result.add createFutureVarCompletions(futureVarIdents) - result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of nnkCommand, nnkCall: @@ -347,6 +354,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = futureVarIdents, nil) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: + procBody.add(createFutureVarCompletions(futureVarIdents, nil)) + if not subtypeIsVoid: procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( @@ -366,8 +375,6 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> complete(retFuture) procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - procBody.add(createFutureVarCompletions(futureVarIdents)) - var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], procBody, nnkIteratorDef) closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) @@ -377,7 +384,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = #var cbName = newIdentNode("cb") var procCb = getAst createCb(retFutureSym, iteratorNameSym, newStrLitNode(prc[0].getName), - createFutureVarCompletions(futureVarIdents)) + createFutureVarCompletions(futureVarIdents, nil)) outerProcBody.add procCb # -> return retFuture @@ -398,7 +405,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = if procBody.kind != nnkEmpty: result[6] = outerProcBody #echo(treeRepr(result)) - #if prc[0].getName == "beta": + #if prc[0].getName == "recvLineInto": # echo(toStrLit(result)) macro async*(prc: untyped): untyped = From 172a9c8e97694846c3348983a9b2b7c2931c939d Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 27 Mar 2017 21:14:06 +0200 Subject: [PATCH 3/8] Implement touch event types/procs in DOM module. --- lib/js/dom.nim | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 5a33dfa9f2..a879ddb612 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -130,6 +130,16 @@ type readOnly*: bool options*: seq[OptionElement] + # https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement + HtmlElement* = ref object of Element + contentEditable*: string + isContentEditable*: bool + dir*: string + offsetHeight*: int + offsetWidth*: int + offsetLeft*: int + offsetTop*: int + LinkElement* = ref LinkObj LinkObj {.importc.} = object of ElementObj target*: cstring @@ -270,6 +280,8 @@ type wordSpacing*: cstring zIndex*: int + # TODO: A lot of the fields in Event belong to a more specific type of event. + # TODO: Should we clean this up? Event* = ref EventObj EventObj {.importc.} = object of RootObj target*: Node @@ -310,6 +322,20 @@ type SUBMIT*: int UNLOAD*: int + TouchList* {.importc.} = ref object of RootObj + length*: int + + TouchEvent* {.importc.} = ref object of Event + changedTouches*, targetTouches*, touches*: TouchList + + Touch* {.importc.} = ref object of RootObj + identifier*: int + screenX*, screenY*, clientX*, clientY*, pageX*, pageY*: int + target*: Element + radiusX*, radiusY*: int + rotationAngle*: int + force*: float + Location* = ref LocationObj LocationObj {.importc.} = object of RootObj hash*: cstring @@ -488,6 +514,10 @@ proc setAttribute*(s: Style, attr, value: cstring, caseSensitive=false) # Event "methods" proc preventDefault*(ev: Event) +# TouchEvent "methods" +proc identifiedTouch*(list: TouchList): Touch +proc item*(list: TouchList, i: int): Touch + {.pop.} var From 04ebe7be51a458dfce01fefd0041e2ed618508a7 Mon Sep 17 00:00:00 2001 From: Zion Date: Mon, 27 Mar 2017 23:55:51 -0700 Subject: [PATCH 4/8] Make fonts load from https rather than http --- config/nimdoc.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 832b4ffd01..d2cfc95401 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -133,8 +133,8 @@ doc.file = """ - - + + $title From 3fc75b2ca411a49c9b61a8b023eb167fa452e353 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 29 Mar 2017 09:32:16 +0300 Subject: [PATCH 5/8] Fix tioselectors.nim test timeouts to be executed in appveyor more stable. (#5622) --- tests/async/tioselectors.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index 048111d030..5e5a18a1b3 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -508,8 +508,8 @@ else: freeAddrInfo(aiList) # for some reason Windows select doesn't return both # descriptors from first call, so we need to make 2 calls - discard selector.select(100) - var rcm = selector.select(100) + discard selector.select(1000) + var rcm = selector.select(1000) assert(len(rcm) == 2) var sockAddress = SockAddr() @@ -526,7 +526,7 @@ else: selector.updateHandle(client_socket, {Event.Read}) - var rc2 = selector.select(100) + var rc2 = selector.select(1000) assert(len(rc2) == 1) var read_count = recv(server2_socket, addr buffer[0], 128, 0) @@ -595,7 +595,7 @@ else: proc event_wait_thread(event: SelectEvent) {.thread.} = var selector = newSelector[int]() selector.registerEvent(event, 1) - var rc = selector.select(500) + var rc = selector.select(1500) if len(rc) == 1: inc(counter) selector.unregister(event) From 977758fe062a51f6d6dc2be227688ddfa768c680 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 29 Mar 2017 09:34:07 +0300 Subject: [PATCH 6/8] Fix posix.nim `dirent` structure to be more compatible with OSes. (#5623) --- lib/posix/posix.nim | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index a18db21500..b29f43eb4c 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -109,12 +109,17 @@ type Dirent* {.importc: "struct dirent", header: "", final, pure.} = object ## dirent_t struct d_ino*: Ino ## File serial number. - when defined(linux) or defined(macosx) or defined(bsd): + when defined(dragonfly): + # DragonflyBSD doesn't have `d_reclen` field. + d_type*: uint8 + elif defined(linux) or defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd): d_reclen*: cshort ## Length of this record. (not POSIX) d_type*: int8 ## Type of file; not supported by all filesystem types. ## (not POSIX) - when defined(linux) or defined(bsd): + when defined(linux) or defined(openbsd): d_off*: Off ## Not an offset. Value that ``telldir()`` would return. + d_name*: array[0..255, char] ## Name of entry. Tflock* {.importc: "struct flock", final, pure, From 27a291c09e5b8861fa195e782c9b46d9bd44b77b Mon Sep 17 00:00:00 2001 From: Tomas Vojtisek Date: Wed, 29 Mar 2017 08:36:10 +0200 Subject: [PATCH 7/8] Update threadpool.nim (#5624) --- lib/pure/concurrency/threadpool.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 8cdb83e196..f438a85e76 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -418,7 +418,7 @@ proc spawn*(call: expr): expr {.magic: "Spawn".} proc pinnedSpawn*(id: ThreadId; call: expr): expr {.magic: "Spawn".} ## always spawns a new task on the worker thread with ``id``, so that ## the 'call' is **always** executed on - ## the this thread. 'call' has to be proc call 'p(...)' where 'p' + ## the thread. 'call' has to be proc call 'p(...)' where 'p' ## is gcsafe and has a return type that is either 'void' or compatible ## with ``FlowVar[T]``. From 875e344be0f0202885f0d5ed7f10188835a171d0 Mon Sep 17 00:00:00 2001 From: Konstantin Molchanov Date: Wed, 29 Mar 2017 14:36:04 +0400 Subject: [PATCH 8/8] JS: Add yearday calculation to getLocalTime and getGMTime, so that yearday is not 0 for TimeInfo instances under JS backend. (#5616) --- lib/pure/times.nim | 10 ++++++++-- tests/js/ttimes.nim | 13 +++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/js/ttimes.nim diff --git a/lib/pure/times.nim b/lib/pure/times.nim index fe35c404c6..2b7c221452 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -597,9 +597,12 @@ elif defined(JS): result.month = Month(t.getMonth()) result.year = t.getFullYear() result.weekday = weekDays[t.getDay()] - result.yearday = 0 result.timezone = getTimezone() + result.yearday = result.monthday - 1 + for month in mJan..