add std/tasks (#17447)

changelog.md

@@ -264,6 +264,8 @@
 
 - Added dollar `$` and `len` for `jsre.RegExp`.
 
+- Added `std/tasks`.
+
 - Added `hasDataBuffered` to `asyncnet`.
 
 - Added `hasClosure` to `std/typetraits`.

lib/std/tasks.nim

@@ -0,0 +1,272 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2021 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module provides basic primitives for creating parallel programs.
+## A `Task` should be only owned by a single Thread, it cannot be shared by threads.
+
+import std/[macros, isolation, typetraits]
+import system/ansi_c
+
+export isolation
+
+
+when compileOption("threads"):
+  from std/effecttraits import isGcSafe
+
+
+#
+# proc hello(a: int, b: string) =
+#   echo $a & b
+#
+# let literal = "Nim"
+# let t = toTask(hello(521, literal))
+#
+#
+# is roughly converted to
+#
+# type
+#   ScratchObj_369098780 = object
+#     a: int
+#     b: string
+#
+# let scratch_369098762 = cast[ptr ScratchObj_369098780](c_calloc(csize_t 1,
+#     csize_t sizeof(ScratchObj_369098780)))
+# if scratch_369098762.isNil:
+#   raise newException(OutOfMemDefect, "Could not allocate memory")
+# block:
+#   var isolate_369098776 = isolate(521)
+#   scratch_369098762.a = extract(isolate_369098776)
+#   var isolate_369098778 = isolate(literal)
+#   scratch_369098762.b = extract(isolate_369098778)
+# proc hello_369098781(args`gensym3: pointer) {.nimcall.} =
+#   let objTemp_369098775 = cast[ptr ScratchObj_369098780](args`gensym3)
+#   let :tmp_369098777 = objTemp_369098775.a
+#   let :tmp_369098779 = objTemp_369098775.b
+#   hello(a = :tmp_369098777, b = :tmp_369098779)
+#
+# proc destroyScratch_369098782(args`gensym3: pointer) {.nimcall.} =
+#   let obj_369098783 = cast[ptr ScratchObj_369098780](args`gensym3)
+#   =destroy(obj_369098783[])
+# let t = Task(callback: hello_369098781, args: scratch_369098762, destroy: destroyScratch_369098782)
+#
+
+
+type
+  Task* = object ## `Task` contains the callback and its arguments.
+    callback: proc (args: pointer) {.nimcall.}
+    args: pointer
+    destroy: proc (args: pointer) {.nimcall.}
+
+
+proc `=copy`*(x: var Task, y: Task) {.error.}
+
+proc `=destroy`*(t: var Task) {.inline.} =
+  ## Frees the resources allocated for a `Task`.
+  if t.args != nil:
+    if t.destroy != nil:
+      t.destroy(t.args)
+    c_free(t.args)
+
+proc invoke*(task: Task) {.inline.} =
+  ## Invokes the `task`.
+  task.callback(task.args)
+
+template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) =
+  # block:
+  #   var isoTempA = isolate(521)
+  #   scratch.a = extract(isolateA)
+  #   var isoTempB = isolate(literal)
+  #   scratch.b = extract(isolateB)
+  let isolatedTemp = genSym(nskTemp, "isoTemp")
+  scratchAssignList.add newVarStmt(isolatedTemp, newCall(newidentNode("isolate"), procParam))
+  scratchAssignList.add newAssignment(scratchDotExpr,
+      newcall(newIdentNode("extract"), isolatedTemp))
+
+template addAllNode(assignParam: NimNode, procParam: NimNode) =
+  let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0])
+
+  checkIsolate(scratchAssignList, procParam, scratchDotExpr)
+
+  let tempNode = genSym(kind = nskTemp, ident = formalParams[i][0].strVal)
+  callNode.add nnkExprEqExpr.newTree(formalParams[i][0], tempNode)
+  tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0]))
+  scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam)
+
+macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task =
+  ## Converts the call and its arguments to `Task`.
+  runnableExamples("--gc:orc"):
+    proc hello(a: int) = echo a
+
+    let b = toTask hello(13)
+    assert b is Task
+
+  doAssert getTypeInst(e).typeKind == ntyVoid
+
+  when compileOption("threads"):
+    if not isGcSafe(e[0]):
+      error("'toTask' takes a GC safe call expression")
+
+  if hasClosure(e[0]):
+    error("closure call is not allowed")
+
+  if e.len > 1:
+    let scratchIdent = genSym(kind = nskTemp, ident = "scratch")
+    let impl = e[0].getTypeInst
+
+    when defined(nimTasksDebug):
+      echo impl.treeRepr
+      echo e.treeRepr
+    let formalParams = impl[0]
+
+    var
+      scratchRecList = newNimNode(nnkRecList)
+      scratchAssignList: seq[NimNode]
+      tempAssignList: seq[NimNode]
+      callNode: seq[NimNode]
+
+    let
+      objTemp = genSym(nskTemp, ident = "objTemp")
+
+    for i in 1 ..< formalParams.len:
+      var param = formalParams[i][1]
+
+      if param.kind == nnkBracketExpr and param[0].eqIdent("sink"):
+        param = param[0]
+
+      if param.typeKind in {ntyExpr, ntyStmt}:
+        error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter")
+
+      case param.kind
+      of nnkVarTy:
+        error("'toTask'ed function cannot have a 'var' parameter")
+      of nnkBracketExpr:
+        if param[0].typeKind == ntyTypeDesc:
+          callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i])
+        elif param[0].typeKind in {ntyVarargs, ntyOpenArray}:
+          if param[1].typeKind in {ntyExpr, ntyStmt}:
+            error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter")
+          let
+            seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1])
+            seqCallNode = newcall("@", e[i])
+          addAllNode(seqType, seqCallNode)
+        else:
+          addAllNode(param, e[i])
+      of nnkBracket, nnkObjConstr:
+        # passing by static parameters
+        # so we pass them directly instead of passing by scratchObj
+        callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i])
+      of nnkSym, nnkPtrTy:
+        addAllNode(param, e[i])
+      of nnkCharLit..nnkNilLit:
+        callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i])
+      else:
+        error("not supported type kinds")
+
+    let scratchObjType = genSym(kind = nskType, ident = "ScratchObj")
+    let scratchObj = nnkTypeSection.newTree(
+                      nnkTypeDef.newTree(
+                        scratchObjType,
+                        newEmptyNode(),
+                        nnkObjectTy.newTree(
+                          newEmptyNode(),
+                          newEmptyNode(),
+                          scratchRecList
+                        )
+                      )
+                    )
+
+
+    let scratchObjPtrType = quote do:
+      cast[ptr `scratchObjType`](c_calloc(csize_t 1, csize_t sizeof(`scratchObjType`)))
+
+    let scratchLetSection = newLetStmt(
+      scratchIdent,
+      scratchObjPtrType
+    )
+
+    let scratchCheck = quote do:
+      if `scratchIdent`.isNil:
+        raise newException(OutOfMemDefect, "Could not allocate memory")
+
+    var stmtList = newStmtList()
+    stmtList.add(scratchObj)
+    stmtList.add(scratchLetSection)
+    stmtList.add(scratchCheck)
+    stmtList.add(nnkBlockStmt.newTree(newEmptyNode(), newStmtList(scratchAssignList)))
+
+    var functionStmtList = newStmtList()
+    let funcCall = newCall(e[0], callNode)
+    functionStmtList.add tempAssignList
+    functionStmtList.add funcCall
+
+    let funcName = genSym(nskProc, e[0].strVal)
+    let destroyName = genSym(nskProc, "destroyScratch")
+    let objTemp2 = genSym(ident = "obj")
+    let tempNode = quote("@") do:
+        `=destroy`(@objTemp2[])
+
+    result = quote do:
+      `stmtList`
+
+      proc `funcName`(args: pointer) {.nimcall.} =
+        let `objTemp` = cast[ptr `scratchObjType`](args)
+        `functionStmtList`
+
+      proc `destroyName`(args: pointer) {.nimcall.} =
+        let `objTemp2` = cast[ptr `scratchObjType`](args)
+        `tempNode`
+
+      Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`)
+  else:
+    let funcCall = newCall(e[0])
+    let funcName = genSym(nskProc, e[0].strVal)
+
+    result = quote do:
+      proc `funcName`(args: pointer) {.nimcall.} =
+        `funcCall`
+
+      Task(callback: `funcName`, args: nil)
+
+  when defined(nimTasksDebug):
+    echo result.repr
+
+runnableExamples("--gc:orc"):
+  block:
+    var num = 0
+    proc hello(a: int) = inc num, a
+
+    let b = toTask hello(13)
+    b.invoke()
+    assert num == 13
+    # A task can be invoked multiple times
+    b.invoke()
+    assert num == 26
+
+  block:
+    type
+      Runnable = ref object
+        data: int
+
+    var data: int
+    proc hello(a: Runnable) {.nimcall.} =
+      a.data += 2
+      data = a.data
+
+
+    when false:
+      # the parameters of call must be isolated.
+      let x = Runnable(data: 12)
+      let b = toTask hello(x) # error ----> expression cannot be isolated: x
+      b.invoke()
+
+    let b = toTask(hello(Runnable(data: 12)))
+    b.invoke()
+    assert data == 14
+    b.invoke()
+    assert data == 16

tests/stdlib/ttasks.nim

@@ -0,0 +1,506 @@
+discard """
+  targets: "c cpp"
+  matrix: "--gc:orc"
+"""
+
+import std/[tasks, strformat]
+
+block:
+  var s = ""
+  proc `+`(x: int, y: string) =
+    s.add $x & y
+
+  let literal = "Nim"
+  let t = toTask(521 + literal)
+  t.invoke()
+
+  doAssert s == "521Nim"
+
+block:
+  var s = ""
+  proc `!`(x: int) =
+    s.add $x
+
+  let t = toTask !12
+  t.invoke()
+
+  doAssert s == "12"
+
+
+block:
+  block:
+    var called = 0
+    proc hello(x: static range[1 .. 5]) =
+      called += x
+
+    let b = toTask hello(3)
+    b.invoke()
+    doAssert called == 3
+    b.invoke()
+    doAssert called == 6
+
+  block:
+    var called = 0
+    proc hello(x: range[1 .. 5]) =
+      called += x
+
+    let b = toTask hello(3)
+    b.invoke()
+    doAssert called == 3
+    b.invoke()
+    doAssert called == 6
+
+  block:
+    var called = 0
+    proc hello(x: 1 .. 5) =
+      called += x
+
+    let b = toTask hello(3)
+    b.invoke()
+    doAssert called == 3
+    b.invoke()
+    doAssert called == 6
+
+  block:
+    var temp = ""
+    proc hello(a: int or seq[string]) =
+      when a is seq[string]:
+        for s in a:
+          temp.add s
+      else:
+        temp.addInt a
+
+    let x = @["1", "2", "3", "4"]
+    let b = toTask hello(x)
+    b.invoke()
+    doAssert temp == "1234"
+    b.invoke()
+    doAssert temp == "12341234"
+
+
+  block:
+    var temp = ""
+
+    proc hello(a: int or string) =
+      when a is string:
+        temp.add a
+
+    let x = "!2"
+
+    let b = toTask hello(x)
+    b.invoke()
+    doAssert temp == x
+
+  block:
+    var temp = ""
+    proc hello(a: int or string) =
+      when a is string:
+        temp.add a
+
+    let x = "!2"
+    let b = toTask hello(x)
+    b.invoke()
+    doAssert temp == x
+
+  block:
+    var x = 0
+    proc hello(typ: typedesc) =
+      x += typ(12)
+
+    let b = toTask hello(int)
+    b.invoke()
+    doAssert x == 12
+
+  block:
+    var temp = ""
+    proc hello(a: int or seq[string]) =
+      when a is seq[string]:
+        for s in a:
+          temp.add s
+
+    let x = @["1", "2", "3", "4"]
+    let b = toTask hello(x)
+    b.invoke()
+    doAssert temp == "1234"
+
+  block:
+    var temp = ""
+    proc hello(a: int | string) =
+      when a is string:
+        temp.add a
+
+    let x = "!2"
+    let b = toTask hello(x)
+    b.invoke()
+    doAssert temp == x
+
+  block:
+    var x = 0
+    proc hello(a: int | string) =
+      when a is int:
+        x = a
+
+    let b = toTask hello(12)
+    b.invoke()
+    doAssert x == 12
+
+  block:
+    var a1: seq[int]
+    var a2 = 0
+    proc hello(c: seq[int], a: int) =
+      a1 = c
+      a2 = a
+
+    let x = 12
+    var y = @[1, 3, 1, 4, 5, x, 1]
+    let b = toTask hello(y, 12)
+    b.invoke()
+
+    doAssert a1 == y
+    doAssert a2 == x
+
+  block:
+    var a1: seq[int]
+    var a2 = 0
+    proc hello(c: seq[int], a: int) =
+      a1 = c
+      a2 = a
+    var x = 2
+    let b = toTask hello(@[1, 3, 1, 4, 5, x, 1], 12)
+    b.invoke()
+
+    doAssert a1 == @[1, 3, 1, 4, 5, x, 1]
+    doAssert a2 == 12
+
+  block:
+    var a1: array[7, int]
+    var a2 = 0
+    proc hello(c: array[7, int], a: int) =
+      a1 = c
+      a2 = a
+
+    let b = toTask hello([1, 3, 1, 4, 5, 2, 1], 12)
+    b.invoke()
+
+    doAssert a1 == [1, 3, 1, 4, 5, 2, 1]
+    doAssert a2 == 12
+
+  block:
+    var a1: seq[int]
+    var a2 = 0
+    proc hello(c: seq[int], a: int) =
+      a1 = c
+      a2 = a
+
+    let b = toTask hello(@[1, 3, 1, 4, 5, 2, 1], 12)
+    b.invoke()
+
+    doAssert a1 == @[1, 3, 1, 4, 5, 2, 1]
+    doAssert a2 == 12
+
+  block:
+    var a1: seq[int]
+    var a2 = 0
+    proc hello(a: int, c: seq[int]) =
+      a1 = c
+      a2 = a
+
+    let b = toTask hello(8, @[1, 3, 1, 4, 5, 2, 1])
+    b.invoke()
+
+    doAssert a1 == @[1, 3, 1, 4, 5, 2, 1]
+    doAssert a2 == 8
+
+    let c = toTask 8.hello(@[1, 3, 1, 4, 5, 2, 1])
+    c.invoke()
+
+    doAssert a1 == @[1, 3, 1, 4, 5, 2, 1]
+    doAssert a2 == 8
+
+  block:
+    var a1: seq[seq[int]]
+    var a2: int
+    proc hello(a: int, c: openArray[seq[int]]) =
+      a1 = @c
+      a2 = a
+
+    let b = toTask hello(8, @[@[3], @[4], @[5], @[6], @[12], @[7]])
+    b.invoke()
+
+    doAssert a1 ==  @[@[3], @[4], @[5], @[6], @[12], @[7]]
+    doAssert a2 == 8
+
+  block:
+    var a1: seq[int]
+    var a2: int
+    proc hello(a: int, c: openArray[int]) =
+      a1 = @c
+      a2 = a
+
+    let b = toTask hello(8, @[3, 4, 5, 6, 12, 7])
+    b.invoke()
+
+    doAssert a1 == @[3, 4, 5, 6, 12, 7]
+    doAssert a2 == 8
+
+  block:
+    var a1: seq[int]
+    var a2: int
+    proc hello(a: int, c: static varargs[int]) =
+      a1 = @c
+      a2 = a
+
+    let b = toTask hello(8, @[3, 4, 5, 6, 12, 7])
+    b.invoke()
+
+    doAssert a1 == @[3, 4, 5, 6, 12, 7]
+    doAssert a2 == 8
+
+  block:
+    var a1: seq[int]
+    var a2: int
+    proc hello(a: int, c: static varargs[int]) =
+      a1 = @c
+      a2 = a
+
+    let b = toTask hello(8, [3, 4, 5, 6, 12, 7])
+    b.invoke()
+
+    doAssert a1 == @[3, 4, 5, 6, 12, 7]
+    doAssert a2 == 8
+
+  block:
+    var a1: seq[int]
+    var a2: int
+    proc hello(a: int, c: varargs[int]) =
+      a1 = @c
+      a2 = a
+
+    let x = 12
+    let b = toTask hello(8, 3, 4, 5, 6, x, 7)
+    b.invoke()
+
+    doAssert a1 == @[3, 4, 5, 6, 12, 7]
+    doAssert a2 == 8
+
+  block:
+    var x = 12
+
+    proc hello(x: ptr int) =
+      x[] += 12
+
+    let b = toTask hello(addr x)
+    b.invoke()
+
+    doAssert x == 24
+
+    let c = toTask x.addr.hello
+    invoke(c)
+
+    doAssert x == 36
+  block:
+    type
+      Test = ref object
+        id: int
+
+    var x = 0
+    proc hello(a: int, c: static Test) =
+      x += a
+      x += c.id
+
+    let b = toTask hello(8, Test(id: 12))
+    b.invoke()
+
+    doAssert x == 20
+
+  block:
+    type
+      Test = object
+        id: int
+
+    var x = 0
+    proc hello(a: int, c: static Test) =
+      x += a
+      x += c.id
+
+    let b = toTask hello(8, Test(id: 12))
+    b.invoke()
+    doAssert x == 20
+
+  block:
+    var x = 0
+    proc hello(a: int, c: static seq[int]) =
+      x += a
+      for i in c:
+        x += i
+
+    let b = toTask hello(8, @[3, 4, 5, 6, 12, 7])
+    b.invoke()
+    doAssert x == 45
+
+  block:
+    var x = 0
+    proc hello(a: int, c: static array[5, int]) =
+      x += a
+      for i in c:
+        x += i
+
+    let b = toTask hello(8, [3, 4, 5, 6, 12])
+    b.invoke()
+    doAssert x == 38
+
+  block:
+    var aVal = 0
+    var cVal = ""
+
+    proc hello(a: int, c: static string) =
+      aVal += a
+      cVal.add c
+
+    var x = 1314
+    let b = toTask hello(x, "hello")
+    b.invoke()
+
+    doAssert aVal == x
+    doAssert cVal == "hello"
+
+  block:
+    var aVal = ""
+
+    proc hello(a: static string) =
+      aVal.add a
+    let b = toTask hello("hello")
+    b.invoke()
+
+    doAssert aVal == "hello"
+
+  block:
+    var aVal = 0
+    var cVal = ""
+
+    proc hello(a: static int, c: static string) =
+      aVal += a
+      cVal.add c
+    let b = toTask hello(8, "hello")
+    b.invoke()
+
+    doAssert aVal == 8
+    doAssert cVal == "hello"
+
+  block:
+    var aVal = 0
+    var cVal = 0
+
+    proc hello(a: static int, c: int) =
+      aVal += a
+      cVal += c
+
+    let b = toTask hello(c = 0, a = 8)
+    b.invoke()
+
+    doAssert aVal == 8
+    doAssert cVal == 0
+
+  block:
+    var aVal = 0
+    var cVal = 0
+
+    proc hello(a: int, c: static int) =
+      aVal += a
+      cVal += c
+
+    let b = toTask hello(c = 0, a = 8)
+    b.invoke()
+
+    doAssert aVal == 8
+    doAssert cVal == 0
+
+  block:
+    var aVal = 0
+    var cVal = 0
+
+    proc hello(a: static int, c: static int) =
+      aVal += a
+      cVal += c
+
+    let b = toTask hello(0, 8)
+    b.invoke()
+
+    doAssert aVal == 0
+    doAssert cVal == 8
+
+  block:
+    var temp = ""
+    proc hello(x: int, y: seq[string], d = 134) =
+      temp = fmt"{x=} {y=} {d=}"
+
+
+    proc main() =
+      var x = @["23456"]
+      let t = toTask hello(2233, x)
+      t.invoke()
+
+      doAssert temp == """x=2233 y=@["23456"] d=134"""
+
+    main()
+
+
+  block:
+    var temp = ""
+    proc hello(x: int, y: seq[string], d = 134) =
+      temp.add fmt"{x=} {y=} {d=}"
+
+    proc ok() =
+      temp = "ok"
+
+    proc main() =
+      var x = @["23456"]
+      let t = toTask hello(2233, x)
+      t.invoke()
+      t.invoke()
+
+      doAssert temp == """x=2233 y=@["23456"] d=134x=2233 y=@["23456"] d=134"""
+
+    main()
+
+    var x = @["4"]
+    let m = toTask hello(2233, x, 7)
+    m.invoke()
+
+    doAssert temp == """x=2233 y=@["23456"] d=134x=2233 y=@["23456"] d=134x=2233 y=@["4"] d=7"""
+
+    let n = toTask ok()
+    n.invoke()
+
+    doAssert temp == "ok"
+
+  block:
+    var called = 0
+    block:
+      proc hello() =
+        inc called
+
+      let a = toTask hello()
+      invoke(a)
+
+    doAssert called == 1
+
+    block:
+      proc hello(a: int) =
+        inc called, a
+
+      let b = toTask hello(13)
+      let c = toTask hello(a = 14)
+      b.invoke()
+      c.invoke()
+
+    doAssert called == 28
+
+    block:
+      proc hello(a: int, c: int) =
+        inc called, a
+
+      let b = toTask hello(c = 0, a = 8)
+      b.invoke()
+
+    doAssert called == 36