fixes #11844 (#11935)

(cherry picked from commit 433613e267)

changelog_v020.md

@@ -1,39 +1,20 @@
-# v0.20.2 - XXXX-XX-XX
+# v0.20.4 - xxxx-xx-xx
 
 
 ## Changes affecting backwards compatibility
 
-- All `strutils.rfind` procs now take `start` and `last` like `strutils.find`
-  with the same data slice/index meaning. This is backwards compatible for
-  calls *not* changing the `rfind` `start` parameter from its default. (#11487)
-
-  In the unlikely case that you were using `rfind X, start=N`, or `rfind X, N`,
-  then you need to change that to `rfind X, last=N` or `rfind X, 0, N`. (This
-  should minimize gotchas porting code from other languages like Python or C++.)
-
-- On Windows stderr/stdout/stdin are not opened as binary files anymore. Use the switch
-  `-d:nimBinaryStdFiles` for a transition period.
 
 ### Breaking changes in the standard library
 
-- Mac OS X / BSD: TSa_Family is now the ``uint8`` type, so type
-  conversions like ``x.sin_family = uint16 toInt(nativesockets.AF_INET)``
-  need to be changed into ``x.sin_family = TSa_Family toInt(nativesockets.AF_INET)``.
-
 
 ### Breaking changes in the compiler
 
 
 ## Library additions
 
-- `toOpenArray` is now available for the JS target.
 
 ## Library changes
 
-- Fix async IO operations stalling even after socket is closed. (#11232)
-
-- More informative error message for `streams.openFileStream`. (#11438)
-
 
 ## Language additions
 
@@ -46,10 +27,5 @@
 
 ### Compiler changes
 
-- Better error message for IndexError for empty containers. (#11476)
-
-- Fix regression in semfold for old right shift. (#11477)
-
-- Fix for passing tuples as static params to macros. (#11423)
 
 ## Bugfixes

changelogs/changelog_0_20_2.md

@@ -0,0 +1,55 @@
+# v0.20.2 - 2019-07-17
+
+
+## Changes affecting backwards compatibility
+
+- All `strutils.rfind` procs now take `start` and `last` like `strutils.find`
+  with the same data slice/index meaning. This is backwards compatible for
+  calls *not* changing the `rfind` `start` parameter from its default. (#11487)
+
+  In the unlikely case that you were using `rfind X, start=N`, or `rfind X, N`,
+  then you need to change that to `rfind X, last=N` or `rfind X, 0, N`. (This
+  should minimize gotchas porting code from other languages like Python or C++.)
+
+- On Windows stderr/stdout/stdin are not opened as binary files anymore. Use the switch
+  `-d:nimBinaryStdFiles` for a transition period.
+
+### Breaking changes in the standard library
+
+- Mac OS X / BSD: TSa_Family is now the ``uint8`` type, so type
+  conversions like ``x.sin_family = uint16 toInt(nativesockets.AF_INET)``
+  need to be changed into ``x.sin_family = TSa_Family toInt(nativesockets.AF_INET)``.
+
+
+### Breaking changes in the compiler
+
+
+## Library additions
+
+- `toOpenArray` is now available for the JS target.
+
+## Library changes
+
+- Fix async IO operations stalling even after socket is closed. (#11232)
+
+- More informative error message for `streams.openFileStream`. (#11438)
+
+
+## Language additions
+
+
+## Language changes
+
+
+### Tool changes
+
+
+### Compiler changes
+
+- Better error message for IndexError for empty containers. (#11476)
+
+- Fix regression in semfold for old right shift. (#11477)
+
+- Fix for passing tuples as static params to macros. (#11423)
+
+## Bugfixes

changelogs/changelog_X_XX_X.md

@@ -1,27 +1,33 @@
-## v0.XX.0 - XX/XX/2018
-
-### Changes affecting backwards compatibility
-
-- Example item: ``Foo`` changed to ``Bar``.
-
-#### Breaking changes in the standard library
+# vx.xx.x - yyyy-mm-dd
 
 
-#### Breaking changes in the compiler
+## Changes affecting backwards compatibility
 
-### Library additions
-
-### Library changes
+- Example item: `Foo` changed to `Bar`.
 
 
-### Language additions
+### Breaking changes in the standard library
 
 
-### Language changes
+### Breaking changes in the compiler
+
+
+## Library additions
+
+
+## Library changes
+
+
+## Language additions
+
+
+## Language changes
 
 
 ### Tool changes
 
+
 ### Compiler changes
 
-### Bugfixes
+
+## Bugfixes

compiler/aliases.nim

@@ -10,7 +10,7 @@
 ## Simple alias analysis for the HLO and the code generators.
 
 import
-  ast, astalgo, types, trees, intsets, msgs
+  ast, astalgo, types, trees, intsets
 
 type
   TAnalysisResult* = enum

compiler/ast.nim

@@ -10,8 +10,7 @@
 # abstract syntax tree + symbol table
 
 import
-  lineinfos, hashes, nversion, options, strutils, std / sha1, ropes, idents,
-  intsets, idgen
+  lineinfos, hashes, options, ropes, idents, idgen
 
 type
   TCallingConvention* = enum
@@ -758,7 +757,9 @@ type
     lfImportCompilerProc,     # ``importc`` of a compilerproc
     lfSingleUse               # no location yet and will only be used once
     lfEnforceDeref            # a copyMem is required to dereference if this a
-      # ptr array due to C array limitations. See #1181, #6422, #11171
+                              # ptr array due to C array limitations.
+                              # See #1181, #6422, #11171
+    lfPrepareForMutation      # string location is about to be mutated (V2)
   TStorageLoc* = enum
     OnUnknown,                # location is unknown (stack, heap or static)
     OnStatic,                 # in a static section

compiler/ccgexprs.nim

@@ -941,6 +941,10 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   if d.k == locNone: d.storage = OnHeap
   if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}:
     a.r = ropecg(p.module, "(*$1)", [a.r])
+
+  if lfPrepareForMutation in d.flags and ty.kind == tyString and
+      p.config.selectedGC == gcDestructors:
+    linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
   putIntoDest(p, d, n,
               ropecg(p.module, "$1$3[$2]", [rdLoc(a), rdCharLoc(b), dataField(p)]), a.storage)
 
@@ -2709,9 +2713,14 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
   of tyBool: result = rope"NIM_FALSE"
   of tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64: result = rope"0"
   of tyFloat..tyFloat128: result = rope"0.0"
-  of tyCString, tyString, tyVar, tyLent, tyPointer, tyPtr, tySequence, tyUntyped,
+  of tyCString, tyVar, tyLent, tyPointer, tyPtr, tyUntyped,
      tyTyped, tyTypeDesc, tyStatic, tyRef, tyNil:
     result = rope"NIM_NIL"
+  of tyString, tySequence:
+    if p.config.selectedGC == gcDestructors:
+      result = rope"{0, NIM_NIL}"
+    else:
+      result = rope"NIM_NIL"
   of tyProc:
     if t.callConv != ccClosure:
       result = rope"NIM_NIL"

compiler/ccgmerge.nim

@@ -11,8 +11,8 @@
 ## is needed for incremental compilation.
 
 import
-  ast, astalgo, ropes, options, strutils, nimlexbase, msgs, cgendata, rodutils,
-  intsets, platform, llstream, tables, sighashes, modulegraphs, pathutils
+  ast, ropes, options, strutils, nimlexbase, cgendata, rodutils,
+  intsets, llstream, tables, modulegraphs, pathutils
 
 # Careful! Section marks need to contain a tabulator so that they cannot
 # be part of C string literals.

compiler/ccgstmts.nim

@@ -1266,11 +1266,13 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     discard getTypeDesc(p.module, le.typ.skipTypes(skipPtrs))
     initLoc(a, locNone, le, OnUnknown)
     a.flags.incl(lfEnforceDeref)
+    a.flags.incl(lfPrepareForMutation)
     expr(p, le, a)
+    a.flags.excl(lfPrepareForMutation)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)
     assert(a.t != nil)
     genLineDir(p, ri)
-    loadInto(p, e.sons[0], ri, a)
+    loadInto(p, le, ri, a)
   else:
     genLineDir(p, e)
     asgnFieldDiscriminant(p, e)

compiler/ccgutils.nim

@@ -10,7 +10,7 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, astalgo, ropes, hashes, strutils, types, msgs, wordrecg,
+  ast, hashes, strutils, msgs, wordrecg,
   platform, trees, options
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =

compiler/cgen.nim

@@ -11,9 +11,9 @@
 
 import
   ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
-  nversion, nimsets, msgs, std / sha1, bitsets, idents, types,
+  nversion, nimsets, msgs, bitsets, idents, types,
   ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
-  condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
+  rodutils, renderer, cgendata, ccgmerge, aliases,
   lowerings, tables, sets, ndi, lineinfos, pathutils, transf, enumtostr
 
 when not defined(leanCompiler):
@@ -1105,7 +1105,8 @@ proc genProcNoForward(m: BModule, prc: PSym) =
     # a check for ``m.declaredThings``.
     if not containsOrIncl(m.declaredThings, prc.id):
       #if prc.loc.k == locNone:
-      # mangle the inline proc based on the module where it is defined - not on the first module that uses it
+      # mangle the inline proc based on the module where it is defined -
+      # not on the first module that uses it
       fillProcLoc(findPendingModule(m, prc), prc.ast[namePos])
       #elif {sfExportc, sfImportc} * prc.flags == {}:
       #  # reset name to restore consistency in case of hashing collisions:
@@ -1778,10 +1779,6 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
                 else: AbsoluteFile""
   open(result.ndi, ndiName, g.config)
 
-proc nullify[T](arr: var T) =
-  for i in low(arr)..high(arr):
-    arr[i] = Rope(nil)
-
 proc rawNewModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
   result = rawNewModule(g, module, AbsoluteFile toFullPath(conf, module.position.FileIndex))
 
@@ -1872,7 +1869,9 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
   result = n
   if b == nil: return
   var m = BModule(b)
-  if passes.skipCodegen(m.config, n): return
+  if passes.skipCodegen(m.config, n) or
+      not moduleHasChanged(m.g.graph, m.module):
+    return
   m.initProc.options = initProcOptions(m)
   #softRnl = if optLineDir in m.config.options: noRnl else: rnl
   # XXX replicate this logic!
@@ -1883,10 +1882,12 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
     genStmts(m.initProc, transformedN)
 
 proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
-  result = true
   if optForceFullMake notin m.config.globalOptions:
-    if not equalsFile(code, cfile.cname):
-      if m.config.symbolFiles == readOnlySf: #isDefined(m.config, "nimdiff"):
+    if not moduleHasChanged(m.g.graph, m.module):
+      result = false
+    elif not equalsFile(code, cfile.cname):
+      if false:
+        #m.config.symbolFiles == readOnlySf: #isDefined(m.config, "nimdiff"):
         if fileExists(cfile.cname):
           copyFile(cfile.cname.string, cfile.cname.string & ".backup")
           echo "diff ", cfile.cname.string, ".backup ", cfile.cname.string
@@ -1894,12 +1895,15 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
           echo "new file ", cfile.cname.string
       if not writeRope(code, cfile.cname):
         rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
-      return
-    if fileExists(cfile.obj) and os.fileNewer(cfile.obj.string, cfile.cname.string):
+      result = true
+    elif fileExists(cfile.obj) and os.fileNewer(cfile.obj.string, cfile.cname.string):
       result = false
+    else:
+      result = true
   else:
     if not writeRope(code, cfile.cname):
       rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
+    result = true
 
 # We need 2 different logics here: pending modules (including
 # 'nim__dat') may require file merging for the combination of dead code
@@ -1911,18 +1915,19 @@ proc writeModule(m: BModule, pending: bool) =
   let cfile = getCFile(m)
 
   if true or optForceFullMake in m.config.globalOptions:
-    genInitCode(m)
-    finishTypeDescriptions(m)
-    if sfMainModule in m.module.flags:
-      # generate main file:
-      genMainProc(m)
-      add(m.s[cfsProcHeaders], m.g.mainModProcs)
-      generateThreadVarsSize(m)
+    if moduleHasChanged(m.g.graph, m.module):
+      genInitCode(m)
+      finishTypeDescriptions(m)
+      if sfMainModule in m.module.flags:
+        # generate main file:
+        genMainProc(m)
+        add(m.s[cfsProcHeaders], m.g.mainModProcs)
+        generateThreadVarsSize(m)
 
     var cf = Cfile(nimname: m.module.name.s, cname: cfile,
                    obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
     var code = genModule(m, cf)
-    if code != nil:
+    if code != nil or m.config.symbolFiles != disabledSf:
       when hasTinyCBackend:
         if conf.cmd == cmdRun:
           tccgen.compileCCode($code)
@@ -1979,46 +1984,47 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
     for destructorCall in graph.globalDestructors:
       n.add destructorCall
   if passes.skipCodegen(m.config, n): return
-  # if the module is cached, we don't regenerate the main proc
-  # nor the dispatchers? But if the dispatchers changed?
-  # XXX emit the dispatchers into its own .c file?
-  if n != nil:
-    m.initProc.options = initProcOptions(m)
-    genStmts(m.initProc, n)
+  if moduleHasChanged(graph, m.module):
+    # if the module is cached, we don't regenerate the main proc
+    # nor the dispatchers? But if the dispatchers changed?
+    # XXX emit the dispatchers into its own .c file?
+    if n != nil:
+      m.initProc.options = initProcOptions(m)
+      genStmts(m.initProc, n)
 
-  if m.hcrOn:
-    # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init)
-    discard cgsym(m, "programResult")
-    if m.inHcrInitGuard:
-      endBlock(m.initProc)
-
-  if sfMainModule in m.module.flags:
     if m.hcrOn:
-      # pull ("define" since they are inline when HCR is on) these functions in the main file
-      # so it can load the HCR runtime and later pass the library handle to the HCR runtime which
-      # will in turn pass it to the other modules it initializes so they can initialize the
-      # register/get procs so they don't have to have the definitions of these functions as well
-      discard cgsym(m, "nimLoadLibrary")
-      discard cgsym(m, "nimLoadLibraryError")
-      discard cgsym(m, "nimGetProcAddr")
-      discard cgsym(m, "procAddrError")
-      discard cgsym(m, "rawWrite")
+      # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init)
+      discard cgsym(m, "programResult")
+      if m.inHcrInitGuard:
+        endBlock(m.initProc)
 
-    # raise dependencies on behalf of genMainProc
-    if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
-      discard cgsym(m, "initStackBottomWith")
-    if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
-      discard cgsym(m, "initThreadVarsEmulation")
+    if sfMainModule in m.module.flags:
+      if m.hcrOn:
+        # pull ("define" since they are inline when HCR is on) these functions in the main file
+        # so it can load the HCR runtime and later pass the library handle to the HCR runtime which
+        # will in turn pass it to the other modules it initializes so they can initialize the
+        # register/get procs so they don't have to have the definitions of these functions as well
+        discard cgsym(m, "nimLoadLibrary")
+        discard cgsym(m, "nimLoadLibraryError")
+        discard cgsym(m, "nimGetProcAddr")
+        discard cgsym(m, "procAddrError")
+        discard cgsym(m, "rawWrite")
 
-    if m.g.breakpoints != nil:
-      discard cgsym(m, "dbgRegisterBreakpoint")
-    if optEndb in m.config.options:
-      discard cgsym(m, "dbgRegisterFilename")
+      # raise dependencies on behalf of genMainProc
+      if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
+        discard cgsym(m, "initStackBottomWith")
+      if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
+        discard cgsym(m, "initThreadVarsEmulation")
 
-    if m.g.forwardedProcs.len == 0:
-      incl m.flags, objHasKidsValid
-    let disp = generateMethodDispatchers(graph)
-    for x in disp: genProcAux(m, x.sym)
+      if m.g.breakpoints != nil:
+        discard cgsym(m, "dbgRegisterBreakpoint")
+      if optEndb in m.config.options:
+        discard cgsym(m, "dbgRegisterFilename")
+
+      if m.g.forwardedProcs.len == 0:
+        incl m.flags, objHasKidsValid
+      let disp = generateMethodDispatchers(graph)
+      for x in disp: genProcAux(m, x.sym)
 
   m.g.modulesClosed.add m
 

compiler/cgendata.nim

@@ -10,7 +10,7 @@
 ## This module contains the data structures for the C code generation phase.
 
 import
-  ast, astalgo, ropes, passes, options, intsets, platform, sighashes,
+  ast, ropes, options, intsets,
   tables, ndi, lineinfos, pathutils, modulegraphs
 
 type

compiler/cgmeth.nim

@@ -10,7 +10,7 @@
 ## This module implements code generation for multi methods.
 
 import
-  intsets, options, ast, astalgo, msgs, idents, renderer, types, magicsys,
+  intsets, options, ast, msgs, idents, renderer, types, magicsys,
   sempass2, strutils, modulegraphs, lineinfos
 
 proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =

compiler/closureiters.nim

@@ -131,8 +131,8 @@
 #   break :stateLoop
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, idents,
-  renderer, types, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  ast, msgs, idents,
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
 
 type
   Ctx = object

compiler/commands.nim

@@ -27,9 +27,11 @@ bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
 import
   os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
-  wordrecg, parseutils, nimblecmd, idents, parseopt, sequtils, lineinfos,
+  wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
   pathutils, strtabs
 
+from incremental import nimIncremental
+
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
 bootSwitch(usedNativeStacktrace,
@@ -48,15 +50,16 @@ const
       "Compiled at $4\n" &
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
+proc genFeatureDesc[T: enum](t: typedesc[T]): string {.compileTime.} =
+  var x = ""
+  for f in low(T)..high(T):
+    if x.len > 0: x.add "|"
+    x.add $f
+  x
+
 const
   Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
-  FeatureDesc = block:
-    var x = ""
-    for f in low(Feature)..high(Feature):
-      if x.len > 0: x.add "|"
-      x.add $f
-    x
-  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % FeatureDesc
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -669,7 +672,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     helpOnError(conf, pass)
   of "symbolfiles": discard "ignore for backwards compat"
   of "incremental":
-    when not defined(nimIncremental):
+    when not nimIncremental:
       localError(conf, info, "the compiler was not built with " &
         "incremental compilation features; bootstrap with " &
         "-d:nimIncremental to enable")
@@ -742,6 +745,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         conf.features.incl parseEnum[Feature](arg)
       except ValueError:
         localError(conf, info, "unknown experimental feature")
+  of "legacy":
+    try:
+      conf.legacyFeatures.incl parseEnum[LegacyFeature](arg)
+    except ValueError:
+      localError(conf, info, "unknown obsolete feature")
   of "nocppexceptions":
     expectNoArg(conf, switch, arg, pass, info)
     incl(conf.globalOptions, optNoCppExceptions)

compiler/condsyms.nim

@@ -10,7 +10,7 @@
 # This module handles the conditional symbols.
 
 import
-  strtabs, platform, strutils, idents
+  strtabs
 
 from options import Feature
 from lineinfos import HintsToStr, WarningsToStr

compiler/depends.nim

@@ -10,8 +10,7 @@
 # This module implements a dependency file generator.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, modulepaths,
-  pathutils
+  options, ast, ropes, idents, passes, modulepaths, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 

compiler/dfa.nim

@@ -29,7 +29,7 @@
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, astalgo, types, intsets, tables, msgs, options, lineinfos, renderer
+import ast, types, intsets, lineinfos, renderer
 
 from patterns import sameTrees
 

compiler/docgen.nim

@@ -15,8 +15,8 @@ import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
   wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
   packages/docutils/rst, packages/docutils/rstgen,
-  packages/docutils/highlite, json, xmltree, cgi, trees, types,
-  typesrenderer, astalgo, modulepaths, lineinfos, sequtils, intsets,
+  json, xmltree, cgi, trees, types,
+  typesrenderer, astalgo, lineinfos, intsets,
   pathutils, trees
 
 const

compiler/docgen2.nim

@@ -11,8 +11,7 @@
 # semantic checking.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, docgen, lineinfos,
-  pathutils
+  options, ast, msgs, idents, passes, docgen, lineinfos, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 

compiler/evaltempl.nim

@@ -10,8 +10,7 @@
 ## Template evaluation engine. Now hygienic.
 
 import
-  strutils, options, ast, astalgo, msgs, os, idents, wordrecg, renderer,
-  lineinfos
+  strutils, options, ast, astalgo, msgs, renderer, lineinfos
 
 type
   TemplCtx = object

compiler/filter_tmpl.nim

@@ -10,8 +10,8 @@
 # This module implements Nim's standard template filter.
 
 import
-  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
-  renderer, filters, lineinfos, pathutils
+  llstream, strutils, ast, msgs, options,
+  filters, lineinfos, pathutils
 
 type
   TParseState = enum

compiler/filters.nim

@@ -10,7 +10,7 @@
 # This module implements Nim's simple filters and helpers for filters.
 
 import
-  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
+  llstream, idents, strutils, ast, msgs, options,
   renderer, pathutils
 
 proc invalidPragma(conf: ConfigRef; n: PNode) =

compiler/gorgeimpl.nim

@@ -9,7 +9,7 @@
 
 ## Module that implements ``gorge`` for the compiler.
 
-import msgs, std / sha1, os, osproc, streams, strutils, options,
+import msgs, std / sha1, os, osproc, streams, options,
   lineinfos, pathutils
 
 proc readOutput(p: Process): (string, int) =

compiler/idents.nim

@@ -12,7 +12,7 @@
 # id. This module is essential for the compiler's performance.
 
 import
-  hashes, strutils, wordrecg
+  hashes, wordrecg
 
 type
   TIdObj* = object of RootObj

compiler/idgen.nim

@@ -9,7 +9,7 @@
 
 ## This module contains a simple persistent id generator.
 
-import idents, strutils, os, options, pathutils
+import idents, strutils, options, pathutils
 
 var gFrontEndId*: int
 

compiler/importer.nim

@@ -10,8 +10,8 @@
 ## This module implements the symbol importing mechanism.
 
 import
-  intsets, strutils, os, ast, astalgo, msgs, options, idents, lookups,
-  semdata, passes, renderer, modulepaths, sigmatch, lineinfos
+  intsets, ast, astalgo, msgs, options, idents, lookups,
+  semdata, modulepaths, sigmatch, lineinfos
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}

compiler/incremental.nim

@@ -10,13 +10,14 @@
 ## Basic type definitions the module graph needs in order to support
 ## incremental compilations.
 
-const nimIncremental* = defined(nimIncremental)
+const nimIncremental* = true # defined(nimIncremental)
 
-import options, lineinfos, pathutils
+import options, lineinfos
 
 when nimIncremental:
-  import ast, msgs, intsets, btrees, db_sqlite, std / sha1
+  import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
   from strutils import parseInt
+  from os import isAbsolute
 
   type
     Writer* = object
@@ -47,7 +48,7 @@ when nimIncremental:
 
   proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
     result = msgs.getHash(conf, fileIdx)
-    if result.len == 0:
+    if result.len == 0 and isAbsolute(string fullpath):
       result = $secureHashFile(string fullpath)
       msgs.setHash(conf, fileIdx, result)
 

compiler/injectdestructors.nim

@@ -134,7 +134,7 @@ to do it.
 ]#
 
 import
-  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
+  intsets, ast, msgs, renderer, magicsys, types, idents,
   strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
   lineinfos, parampatterns, sighashes
 
@@ -437,17 +437,6 @@ proc sinkParamIsLastReadCheck(c: var Con, s: PNode) =
      localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s &
          "` is already consumed at " & toFileLineCol(c. graph.config, s.info))
 
-proc isSinkTypeForParam(t: PType): bool =
-  # a parameter like 'seq[owned T]' must not be used only once, but its
-  # elements must, so we detect this case here:
-  result = t.skipTypes({tyGenericInst, tyAlias}).kind in {tySink, tyOwned}
-  when false:
-    if isSinkType(t):
-      if t.skipTypes({tyGenericInst, tyAlias}).kind in {tyArray, tyVarargs, tyOpenArray, tySequence}:
-        result = false
-      else:
-        result = true
-
 proc passCopyToSink(n: PNode; c: var Con): PNode =
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
   let tmp = getTemp(c, n.typ, n.info)
@@ -807,7 +796,11 @@ proc p(n: PNode; c: var Con): PNode =
       result = n
   of nkAsgn, nkFastAsgn:
     if hasDestructor(n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda}:
-      result = moveOrCopy(n[0], n[1], c)
+      # rule (self-assignment-removal):
+      if n[1].kind == nkSym and n[0].kind == nkSym and n[0].sym == n[1].sym:
+        result = newNodeI(nkEmpty, n.info)
+      else:
+        result = moveOrCopy(n[0], n[1], c)
     else:
       result = copyNode(n)
       recurse(n, result)

compiler/installer.ini

@@ -86,6 +86,7 @@ Files: "bin/nimsuggest.exe"
 Files: "bin/nimble.exe"
 Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
+Files: "bin/nimpretty.exe"
 
 Files: "koch.exe"
 Files: "finish.exe"

compiler/jsgen.nim

@@ -29,11 +29,11 @@ implements the required case distinction.
 
 
 import
-  ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options,
-  nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os, tables,
-  times, ropes, math, passes, ccgutils, wordrecg, renderer,
+  ast, strutils, trees, magicsys, options,
+  nversion, msgs, idents, types, tables,
+  ropes, math, passes, ccgutils, wordrecg, renderer,
   intsets, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
-  pathutils, transf
+  transf
 
 
 from modulegraphs import ModuleGraph, PPassContext

compiler/lambdalifting.nim

@@ -10,7 +10,7 @@
 # This file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs,
+  intsets, strutils, options, ast, astalgo, msgs,
   idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos,
   transf, liftdestructors
 

compiler/layouter.nim

@@ -32,7 +32,7 @@ type
     ltTab,
     ltOptionalNewline, ## optional newline introduced by nimpretty
     ltComment, ltLit, ltKeyword, ltExportMarker, ltIdent,
-    ltOther, ltOpr,
+    ltOther, ltOpr, ltSomeParLe, ltSomeParRi,
     ltBeginSection, ltEndSection
 
   Emitter* = object
@@ -142,7 +142,7 @@ proc optionalIsGood(em: var Emitter; pos, currentLen: int): bool =
       result = true
     elif em.tokens[p+1].len < ourIndent:
       result = isLongEnough(lineLen, pos, p)
-  elif em.kinds[pos+1] == ltOther: # note: pos+1, not p+1
+  elif em.kinds[pos+1] in {ltOther, ltSomeParLe, ltSomeParRi}: # note: pos+1, not p+1
     result = false
   else:
     result = isLongEnough(lineLen, pos, p)
@@ -153,13 +153,28 @@ proc lenOfNextTokens(em: Emitter; pos: int): int =
     if em.kinds[pos+i] in {ltCrucialNewline, ltSplittingNewline, ltOptionalNewline}: break
     inc result, em.tokens[pos+i].len
 
+proc guidingInd(em: Emitter; pos: int): int =
+  var i = pos - 1
+  while i >= 0 and em.kinds[i] != ltSomeParLe:
+    dec i
+  while i+1 <= em.kinds.high and em.kinds[i] != ltSomeParRi:
+    if em.kinds[i] == ltSplittingNewline and em.kinds[i+1] == ltSpaces:
+      return em.tokens[i+1].len
+    inc i
+  result = -1
+
 proc closeEmitter*(em: var Emitter) =
+  template defaultCase() =
+    content.add em.tokens[i]
+    inc lineLen, em.tokens[i].len
+
   let outFile = em.config.absOutFile
 
   var content = newStringOfCap(16_000)
   var maxLhs = 0
   var lineLen = 0
   var lineBegin = 0
+  var openPars = 0
   var i = 0
   while i <= em.tokens.high:
     when defined(debug):
@@ -201,8 +216,13 @@ proc closeEmitter*(em: var Emitter) =
           let spaces = em.tokens[i-1].len
           content.setLen(content.len - spaces)
         content.add "\L"
-        content.add em.tokens[i]
-        lineLen = em.tokens[i].len
+        let guide = if openPars > 0: guidingInd(em, i) else: -1
+        if guide >= 0:
+          content.add repeat(' ', guide)
+          lineLen = guide
+        else:
+          content.add em.tokens[i]
+          lineLen = em.tokens[i].len
         lineBegin = i+1
         if i+1 < em.kinds.len and em.kinds[i+1] == ltSpaces:
           # inhibit extra spaces at the start of a new line
@@ -215,9 +235,15 @@ proc closeEmitter*(em: var Emitter) =
       else:
         inc lineLen, em.tokens[i].len
       content.add em.tokens[i]
+    of ltSomeParLe:
+      inc openPars
+      defaultCase()
+    of ltSomeParRi:
+      doAssert openPars > 0
+      dec openPars
+      defaultCase()
     else:
-      content.add em.tokens[i]
-      inc lineLen, em.tokens[i].len
+      defaultCase()
     inc i
 
   if fileExists(outFile) and readFile(outFile.string) == content:
@@ -292,7 +318,7 @@ const
               tkCurlyLe}
   closedPars = {tkParRi, tkParDotRi,
               tkBracketRi, tkCurlyDotRi,
-              tkCurlyRi}
+              tkCurlyRi, tkBracketDotRi}
 
   splitters = openPars + {tkComma, tkSemiColon} # do not add 'tkColon' here!
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
@@ -416,7 +442,8 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     var newlineKind = ltCrucialNewline
     if em.keepIndents > 0:
       em.indentLevel = tok.indent
-    elif (em.lastTok in (splitters + oprSet) and tok.tokType notin closedPars):
+    elif (em.lastTok in (splitters + oprSet) and
+        tok.tokType notin (closedPars - {tkBracketDotRi})):
       # aka: we are in an expression context:
       let alignment = max(tok.indent - em.indentStack[^1], 0)
       em.indentLevel = alignment + em.indentStack.high * em.indWidth
@@ -471,18 +498,14 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     wr(em, TokTypeToStr[tok.tokType], ltOther)
     rememberSplit(splitComma)
     wrSpace em
-  of tkParDotLe, tkParLe, tkBracketDotLe, tkBracketLe,
-     tkCurlyLe, tkCurlyDotLe, tkBracketLeColon:
+  of openPars:
     if tok.strongSpaceA > 0 and not em.endsInWhite and not em.wasExportMarker:
       wrSpace em
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, TokTypeToStr[tok.tokType], ltSomeParLe)
     rememberSplit(splitParLe)
-  of tkParRi,
-     tkBracketRi, tkCurlyRi,
-     tkBracketDotRi,
-     tkCurlyDotRi,
-     tkParDotRi,
-     tkColonColon:
+  of closedPars:
+    wr(em, TokTypeToStr[tok.tokType], ltSomeParRi)
+  of tkColonColon:
     wr(em, TokTypeToStr[tok.tokType], ltOther)
   of tkDot:
     lastTokWasTerse = true

compiler/liftdestructors.nim

@@ -13,7 +13,7 @@
 # Todo:
 # - use openArray instead of array to avoid over-specializations
 
-import modulegraphs, lineinfos, idents, ast, astalgo, renderer, semdata,
+import modulegraphs, lineinfos, idents, ast, renderer, semdata,
   sighashes, lowerings, options, types, msgs, magicsys, tables
 
 type

compiler/liftlocals.nim

@@ -10,7 +10,7 @@
 ## This module implements the '.liftLocals' pragma.
 
 import
-  intsets, strutils, options, ast, astalgo, msgs,
+  strutils, options, ast, msgs,
   idents, renderer, types, lowerings, lineinfos
 
 from pragmas import getPragmaVal

compiler/linter.nim

@@ -9,11 +9,9 @@
 
 ## This module implements the style checker.
 
-import
-  strutils, os, intsets, strtabs
+import strutils
 
-import options, ast, astalgo, msgs, semdata, ropes, idents,
-  lineinfos, pathutils, wordrecg
+import options, ast, msgs, idents, lineinfos, wordrecg
 
 const
   Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}

compiler/llstream.nim

@@ -10,7 +10,7 @@
 ## Low-level streams for high performance.
 
 import
-  strutils, pathutils
+  pathutils
 
 # support '-d:useGnuReadline' for backwards compatibility:
 when not defined(windows) and (defined(useGnuReadline) or defined(useLinenoise)):
@@ -97,7 +97,7 @@ proc continueLine(line: string, inTripleString: bool): bool {.inline.} =
 
 proc countTriples(s: string): int =
   var i = 0
-  while i < s.len:
+  while i+2 < s.len:
     if s[i] == '"' and s[i+1] == '"' and s[i+2] == '"':
       inc result
       inc i, 2

compiler/lookups.nim

@@ -11,7 +11,7 @@
 
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, wordrecg, idgen, nimfix/prettybase, lineinfos, strutils
+  renderer, nimfix/prettybase, lineinfos, strutils
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 

compiler/lowerings.nim

@@ -14,7 +14,6 @@ const
 
 import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
   lineinfos
-from trees import getMagic
 
 proc newDeref*(n: PNode): PNode {.inline.} =
   result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])

compiler/macrocacheimpl.nim

@@ -9,7 +9,7 @@
 
 ## This module implements helpers for the macro cache.
 
-import lineinfos, ast, modulegraphs, vmdef, magicsys
+import lineinfos, ast, modulegraphs, vmdef
 
 proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
   var recorded = newNodeI(nkCommentStmt, info)

compiler/magicsys.nim

@@ -10,7 +10,7 @@
 # Built-in types and compilerprocs are registered here.
 
 import
-  ast, astalgo, hashes, msgs, platform, nversion, times, idents,
+  ast, astalgo, msgs, platform, idents,
   modulegraphs, lineinfos
 
 export createMagic

compiler/main.nim

@@ -13,19 +13,17 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs,
-  os, condsyms, times,
-  wordrecg, sem, semdata, idents, passes, extccomp,
+  llstream, strutils, ast, lexer, syntaxes, options, msgs,
+  condsyms, times,
+  sem, idents, passes, extccomp,
   cgen, json, nversion,
-  platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
-  parser, modules, ccgutils, sigmatch, ropes,
+  platform, nimconf, passaux, depends, vm, idgen,
+  parser, modules,
   modulegraphs, tables, rod, lineinfos, pathutils
 
 when not defined(leanCompiler):
   import jsgen, docgen, docgen2
 
-from magicsys import resetSysTypes
-
 proc semanticPasses(g: ModuleGraph) =
   registerPass g, verbosePass
   registerPass g, semPass

compiler/modulepaths.nim

@@ -8,7 +8,7 @@
 #
 
 import ast, renderer, strutils, msgs, options, idents, os, lineinfos,
-  pathutils, nimblecmd
+  pathutils
 
 when false:
   const

compiler/modules.nim

@@ -10,8 +10,8 @@
 ## Implements the module handling, including the caching of modules.
 
 import
-  ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options,
-  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
+  ast, astalgo, magicsys, msgs, options,
+  idents, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
   lineinfos, pathutils, tables
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
@@ -77,8 +77,6 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     if result == nil:
       result = newModule(graph, fileIdx)
       result.flags = result.flags + flags
-      if sfMainModule in result.flags:
-        graph.config.mainPackageId = result.owner.id
       result.id = id
       registerModule(graph, result)
     else:

compiler/msgs.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  options, strutils, os, tables, ropes, platform, terminal, macros,
+  options, strutils, os, tables, ropes, terminal, macros,
   lineinfos, pathutils
 
 proc toCChar*(c: char; result: var string) =

compiler/nim.nim

@@ -19,9 +19,9 @@ when defined(i386) and defined(windows) and defined(vcc):
   {.link: "../icons/nim-i386-windows-vcc.res".}
 
 import
-  commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
-  extccomp, strutils, os, osproc, platform, main, parseopt,
-  scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper,
+  commands, options, msgs,
+  extccomp, strutils, os, main, parseopt,
+  idents, lineinfos, cmdlinehelper,
   pathutils
 
 include nodejs

compiler/nimconf.nim

@@ -10,7 +10,7 @@
 # This module handles the reading of the config file.
 
 import
-  llstream, nversion, commands, os, strutils, msgs, platform, condsyms, lexer,
+  llstream, commands, os, strutils, msgs, lexer,
   options, idents, wordrecg, strtabs, lineinfos, pathutils
 
 # ---------------- configuration file parser -----------------------------

compiler/nimfix/prettybase.nim

@@ -8,9 +8,7 @@
 #
 
 import strutils except Letters
-import lexbase, streams
 import ".." / [ast, msgs, lineinfos, idents, options, linter]
-from os import splitFile
 
 proc replaceDeprecated*(conf: ConfigRef; info: TLineInfo; oldSym, newSym: PIdent) =
   let line = sourceLine(conf, info)

compiler/nimsets.nim

@@ -10,8 +10,7 @@
 # this unit handles Nim sets; it implements symbolic sets
 
 import
-  ast, astalgo, trees, nversion, lineinfos, platform, bitsets, types, renderer,
-  options
+  ast, astalgo, lineinfos, bitsets, types, options
 
 proc inSet*(s: PNode, elem: PNode): bool =
   assert s.kind == nkCurly

compiler/options.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  os, strutils, strtabs, osproc, sets, lineinfos, platform,
+  os, strutils, strtabs, sets, lineinfos, platform,
   prefixmatches, pathutils
 
 from terminal import isatty
@@ -135,6 +135,12 @@ type
       ## which itself requires `nimble install libffi`, see #10150
       ## Note: this feature can't be localized with {.push.}
 
+  LegacyFeature* = enum
+    allowSemcheckedAstModification,
+      ## Allows to modify a NimNode where the type has already been
+      ## flaged with nfSem. If you actually do this, it will cause
+      ## bugs.
+
   SymbolFilesOption* = enum
     disabledSf, writeOnlySf, readOnlySf, v2Sf
 
@@ -196,6 +202,7 @@ type
     cppDefines*: HashSet[string] # (*)
     headerFile*: string
     features*: set[Feature]
+    legacyFeatures*: set[LegacyFeature]
     arguments*: string ## the arguments to be passed to the program that
                        ## should be run
     ideCmd*: IdeCmd
@@ -314,7 +321,7 @@ proc newConfigRef*(): ConfigRef =
     m: initMsgConfig(),
     evalExpr: "",
     cppDefines: initHashSet[string](),
-    headerFile: "", features: {}, foreignPackageNotes: {hintProcessing, warnUnknownMagic,
+    headerFile: "", features: {}, legacyFeatures: {}, foreignPackageNotes: {hintProcessing, warnUnknownMagic,
     hintQuitCalled, hintExecuting},
     notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1],
     configVars: newStringTable(modeStyleInsensitive),

compiler/parampatterns.nim

@@ -10,7 +10,7 @@
 ## This module implements the pattern matching features for term rewriting
 ## macro support.
 
-import strutils, ast, astalgo, types, msgs, idents, renderer, wordrecg, trees,
+import strutils, ast, types, msgs, idents, renderer, wordrecg, trees,
   options
 
 # we precompile the pattern here for efficiency into some internal

compiler/parser.nim

@@ -27,7 +27,7 @@ when isMainModule:
   outp.close
 
 import
-  llstream, lexer, idents, strutils, ast, astalgo, msgs, options, lineinfos,
+  llstream, lexer, idents, strutils, ast, msgs, options, lineinfos,
   pathutils
 
 when defined(nimpretty):
@@ -1203,6 +1203,13 @@ proc parseFor(p: var TParser): PNode =
   colcom(p, result)
   addSon(result, parseStmt(p))
 
+template nimprettyDontTouch(body) =
+  when defined(nimpretty):
+    inc p.em.keepIndents
+  body
+  when defined(nimpretty):
+    dec p.em.keepIndents
+
 proc parseExpr(p: var TParser): PNode =
   #| expr = (blockExpr
   #|       | ifExpr
@@ -1212,12 +1219,26 @@ proc parseExpr(p: var TParser): PNode =
   #|       | tryExpr)
   #|       / simpleExpr
   case p.tok.tokType:
-  of tkBlock: result = parseBlock(p)
-  of tkIf: result = parseIfExpr(p, nkIfExpr)
-  of tkFor: result = parseFor(p)
-  of tkWhen: result = parseIfExpr(p, nkWhenExpr)
-  of tkCase: result = parseCase(p)
-  of tkTry: result = parseTry(p, isExpr=true)
+  of tkBlock:
+    nimprettyDontTouch:
+      result = parseBlock(p)
+  of tkIf:
+    nimprettyDontTouch:
+      result = parseIfExpr(p, nkIfExpr)
+  of tkFor:
+    nimprettyDontTouch:
+      result = parseFor(p)
+  of tkWhen:
+    nimprettyDontTouch:
+      result = parseIfExpr(p, nkWhenExpr)
+  of tkCase:
+    # Currently we think nimpretty is good enough with case expressions,
+    # so it is allowed to touch them:
+    #nimprettyDontTouch:
+    result = parseCase(p)
+  of tkTry:
+    nimprettyDontTouch:
+      result = parseTry(p, isExpr=true)
   else: result = simpleExpr(p)
 
 proc parseEnum(p: var TParser): PNode

compiler/passaux.nim

@@ -10,7 +10,7 @@
 ## implements some little helper passes
 
 import
-  strutils, ast, astalgo, passes, idents, msgs, options, idgen, lineinfos
+  ast, passes, idents, msgs, options, idgen, lineinfos
 
 from modulegraphs import ModuleGraph, PPassContext
 

compiler/passes.nim

@@ -11,9 +11,9 @@
 ## `TPass` interface.
 
 import
-  strutils, options, ast, astalgo, llstream, msgs, platform, os,
-  condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
-  nimsets, syntaxes, times, idgen, modulegraphs, reorder, rod,
+  options, ast, llstream, msgs,
+  idents,
+  syntaxes, idgen, modulegraphs, reorder, rod,
   lineinfos, pathutils
 
 type
@@ -113,6 +113,19 @@ const
     nkMacroDef, nkConverterDef, nkIteratorDef, nkFuncDef, nkPragma,
     nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt}
 
+proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
+  if sfMainModule in module.flags:
+    graph.config.mainPackageId = module.owner.id
+  # don't be verbose unless the module belongs to the main package:
+  if module.owner.id == graph.config.mainPackageId:
+    graph.config.notes = graph.config.mainPackageNotes
+  else:
+    if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
+    graph.config.notes = graph.config.foreignPackageNotes
+
+proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
+  result = module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
+
 proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
   var
@@ -120,6 +133,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
     a: TPassContextArray
     s: PLLStream
     fileIdx = module.fileIdx
+  prepareConfigNotes(graph, module)
   if module.id < 0:
     # new module caching mechanism:
     for i in 0 ..< graph.passes.len:

compiler/pathutils.nim

@@ -10,7 +10,7 @@
 ## Path handling utilities for Nim. Strictly typed code in order
 ## to avoid the never ending time sink in getting path handling right.
 
-import os, strutils, pathnorm
+import os, pathnorm
 
 type
   AbsoluteFile* = distinct string

compiler/patterns.nim

@@ -11,8 +11,7 @@
 ## macro support.
 
 import
-  ast, astalgo, types, semdata, sigmatch, msgs, idents, aliases, parampatterns,
-  trees
+  ast, types, semdata, sigmatch, idents, aliases, parampatterns, trees
 
 type
   TPatternContext = object

compiler/plugins/itersgen.nim

@@ -9,9 +9,7 @@
 
 ## Plugin to transform an inline iterator into a data structure.
 
-import ".." / [ast, astalgo,
-  magicsys, lookups, semdata,
-  lambdalifting, msgs]
+import ".." / [ast, lookups, semdata, lambdalifting, msgs]
 
 proc iterToProcImpl*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkStmtList, n.info)

compiler/plugins/locals.nim

@@ -9,7 +9,7 @@
 
 ## The builtin 'system.locals' implemented as a plugin.
 
-import ".." / [pluginsupport, ast, astalgo,
+import ".." / [ast, astalgo,
   magicsys, lookups, semdata, lowerings]
 
 proc semLocals*(c: PContext, n: PNode): PNode =

compiler/reorder.nim

@@ -1,7 +1,7 @@
 
 import
-  intsets, ast, idents, algorithm, renderer, parser, os, strutils,
-  sequtils, msgs, modulegraphs, syntaxes, options, modulepaths, tables,
+  intsets, ast, idents, algorithm, renderer, os, strutils,
+  msgs, modulegraphs, syntaxes, options, modulepaths,
   lineinfos
 
 type

compiler/rod.nim

@@ -9,7 +9,7 @@
 
 ## This module implements the canonalization for the various caching mechanisms.
 
-import ast, idgen, lineinfos, msgs, incremental, modulegraphs, pathutils
+import ast, idgen, lineinfos, incremental, modulegraphs, pathutils
 
 when not nimIncremental:
   template setupModuleCache*(g: ModuleGraph) = discard

compiler/rodimpl.nim

@@ -9,7 +9,7 @@
 
 ## This module implements the new compilation cache.
 
-import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
+import strutils, intsets, tables, ropes, db_sqlite, msgs, options,
   renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp,
   btrees, trees, condsyms, nversion, pathutils
 
@@ -17,7 +17,6 @@ import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
 ## - Dependency computation should use *signature* hashes in order to
 ##   avoid recompiling dependent modules.
 ## - Patch the rest of the compiler to do lazy loading of proc bodies.
-## - Patch the C codegen to cache proc bodies and maybe types.
 
 template db(): DbConn = g.incr.db
 
@@ -72,9 +71,12 @@ proc getModuleId(g: ModuleGraph; fileIdx: FileIndex; fullpath: AbsoluteFile): in
       # not changed, so use the cached AST:
       doAssert(result != 0)
       var cycleCheck = initIntSet()
-      if not needsRecompile(g, fileIdx, fullpath, cycleCheck) and not g.incr.configChanged:
-        echo "cached successfully! ", string fullpath
-        return -result
+      if not needsRecompile(g, fileIdx, fullpath, cycleCheck):
+        if not g.incr.configChanged or g.config.symbolFiles == readOnlySf:
+          #echo "cached successfully! ", string fullpath
+          return -result
+      elif g.config.symbolFiles == readOnlySf:
+        internalError(g.config, "file needs to be recompiled: " & (string fullpath))
     db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0])
     db.exec(sql"delete from deps where module = ?", module[0])
     db.exec(sql"delete from types where module = ?", module[0])
@@ -221,22 +223,13 @@ proc encodeType(g: ModuleGraph, t: PType, result: var string) =
   if t.lockLevel.ord != UnspecifiedLockLevel.ord:
     add(result, '\14')
     encodeVInt(t.lockLevel.int16, result)
-  if t.destructor != nil and t.destructor.id != 0:
-    add(result, '\15')
-    encodeVInt(t.destructor.id, result)
-    pushSym(w, t.destructor)
-  if t.deepCopy != nil:
+  for a in t.attachedOps:
     add(result, '\16')
-    encodeVInt(t.deepcopy.id, result)
-    pushSym(w, t.deepcopy)
-  if t.assignment != nil:
-    add(result, '\17')
-    encodeVInt(t.assignment.id, result)
-    pushSym(w, t.assignment)
-  if t.sink != nil:
-    add(result, '\18')
-    encodeVInt(t.sink.id, result)
-    pushSym(w, t.sink)
+    if a == nil:
+      encodeVInt(-1, result)
+    else:
+      encodeVInt(a.id, result)
+      pushSym(w, a)
   for i, s in items(t.methods):
     add(result, '\19')
     encodeVInt(i, result)
@@ -305,7 +298,7 @@ proc encodeSym(g: ModuleGraph, s: PSym, result: var string) =
     pushSym(w, s.owner)
   if s.flags != {}:
     result.add('$')
-    encodeVInt(cast[int32](s.flags), result)
+    encodeVBiggestInt(cast[int64](s.flags), result)
   if s.magic != mNone:
     result.add('@')
     encodeVInt(ord(s.magic), result)
@@ -640,18 +633,13 @@ proc loadType(g; id: int; info: TLineInfo): PType =
   else:
     result.lockLevel = UnspecifiedLockLevel
 
-  if b.s[b.pos] == '\15':
-    inc(b.pos)
-    result.destructor = loadSym(g, decodeVInt(b.s, b.pos), info)
-  if b.s[b.pos] == '\16':
-    inc(b.pos)
-    result.deepCopy = loadSym(g, decodeVInt(b.s, b.pos), info)
-  if b.s[b.pos] == '\17':
-    inc(b.pos)
-    result.assignment = loadSym(g, decodeVInt(b.s, b.pos), info)
-  if b.s[b.pos] == '\18':
-    inc(b.pos)
-    result.sink = loadSym(g, decodeVInt(b.s, b.pos), info)
+  for a in low(result.attachedOps)..high(result.attachedOps):
+    if b.s[b.pos] == '\16':
+      inc(b.pos)
+      let id = decodeVInt(b.s, b.pos)
+      if id >= 0:
+        result.attachedOps[a] = loadSym(g, id, info)
+
   while b.s[b.pos] == '\19':
     inc(b.pos)
     let x = decodeVInt(b.s, b.pos)
@@ -739,7 +727,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
     result.owner = loadSym(g, decodeVInt(b.s, b.pos), result.info)
   if b.s[b.pos] == '$':
     inc(b.pos)
-    result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos)))
+    result.flags = cast[TSymFlags](decodeVBiggestInt(b.s, b.pos))
   if b.s[b.pos] == '@':
     inc(b.pos)
     result.magic = TMagic(decodeVInt(b.s, b.pos))
@@ -772,6 +760,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
     if b.s[b.pos] == '\24':
       inc b.pos
       result.transformedBody = decodeNode(g, b, result.info)
+      #result.transformedBody = nil
   of skModule, skPackage:
     decodeInstantiations(g, b, result.info, result.usedGenerics)
   of skLet, skVar, skField, skForVar:
@@ -785,7 +774,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
 
   if b.s[b.pos] == '(':
     #if result.kind in routineKinds:
-    #  result.ast = decodeNodeLazyBody(b, result.info, result)
+    #  result.ast = nil
     #else:
     result.ast = decodeNode(g, b, result.info)
   if sfCompilerProc in result.flags:
@@ -846,7 +835,7 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
       of "error": localError(g.config, n.info, errUser, n[1].strVal)
       of "compile":
         internalAssert g.config, n.len == 3 and n[2].kind == nkStrLit
-        let cname = AbsoluteFile n[1].strVal,
+        let cname = AbsoluteFile n[1].strVal
         var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
                        obj: AbsoluteFile n[2].strVal,
                        flags: {CfileFlag.External})
@@ -902,6 +891,10 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
       internalAssert g.config, imported.id < 0
   of nkStmtList, nkStmtListExpr:
     for x in n: replay(g, module, x)
+  of nkExportStmt:
+    for x in n:
+      doAssert x.kind == nkSym
+      strTableAdd(module.tab, x.sym)
   else: discard "nothing to do for this node"
 
 proc loadNode*(g: ModuleGraph; module: PSym): PNode =

compiler/rodutils.nim

@@ -46,14 +46,9 @@ proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
   of fcNegInf:
     result = "-INF"
   else:
-    when defined(nimNoArrayToCstringConversion):
-      result = newString(81)
-      let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
-      setLen(result, n)
-    else:
-      var buf: array[0..80, char]
-      discard c_snprintf(buf.cstring, buf.len.uint, "%#.16e%s", f, literalPostfix.cstring)
-      result = $buf.cstring
+    result = newString(81)
+    let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
+    setLen(result, n)
 
 proc encodeStr*(s: string, result: var string) =
   for i in 0 ..< len(s):

compiler/scriptconfig.nim

@@ -11,9 +11,10 @@
 ## language.
 
 import
-  ast, modules, idents, passes, passaux, condsyms,
-  options, nimconf, sem, semdata, llstream, vm, vmdef, commands, msgs,
-  os, times, osproc, wordrecg, strtabs, modulegraphs, lineinfos, pathutils
+  ast, modules, idents, passes, condsyms,
+  options, sem, llstream, vm, vmdef, commands, msgs,
+  os, times, osproc, wordrecg, strtabs, modulegraphs,
+  lineinfos, pathutils
 
 # we support 'cmpIgnoreStyle' natively for efficiency:
 from strutils import cmpIgnoreStyle, contains

compiler/sem.nim

@@ -10,13 +10,13 @@
 # This module implements the semantic checking pass.
 
 import
-  ast, strutils, hashes, options, lexer, astalgo, trees, treetab,
-  wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
-  magicsys, parser, nversion, nimsets, semfold, modulepaths, importer,
+  ast, strutils, options, astalgo, trees,
+  wordrecg, ropes, msgs, idents, renderer, types, platform, math,
+  magicsys, nversion, nimsets, semfold, modulepaths, importer,
   procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
-  lowerings, pluginsupport, plugins/active, rod, lineinfos, strtabs, int128
+  lowerings, plugins/active, rod, lineinfos, strtabs, int128
 
 from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward
 
@@ -523,12 +523,6 @@ proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
   if sfSystemModule in module.flags:
     graph.systemModule = module
   c.topLevelScope = openScope(c)
-  # don't be verbose unless the module belongs to the main package:
-  if module.owner.id == graph.config.mainPackageId:
-    graph.config.notes = graph.config.mainPackageNotes
-  else:
-    if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
-    graph.config.notes = graph.config.foreignPackageNotes
   result = c
 
 proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool =

compiler/semcall.nim

@@ -107,7 +107,6 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
       elif errorsEnabled or z.diagnosticsEnabled:
         errors.add(CandidateError(
           sym: sym,
-          unmatchedVarParam: int z.mutabilityProblem,
           firstMismatch: z.firstMismatch,
           diagnostics: z.diagnostics))
     else:
@@ -173,14 +172,14 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
   var filterOnlyFirst = false
   if optShowAllMismatches notin c.config.globalOptions:
     for err in errors:
-      if err.firstMismatch > 1:
+      if err.firstMismatch.arg > 1:
         filterOnlyFirst = true
         break
 
   var candidates = ""
   var skipped = 0
   for err in errors:
-    if filterOnlyFirst and err.firstMismatch == 1:
+    if filterOnlyFirst and err.firstMismatch.arg == 1:
       inc skipped
       continue
     if err.sym.kind in routineKinds and err.sym.ast != nil:
@@ -189,34 +188,35 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
     else:
       add(candidates, getProcHeader(c.config, err.sym, prefer))
     add(candidates, "\n")
-    if err.firstMismatch != 0 and n.len > 1:
-      let cond = n.len > 2
-      if cond:
-        candidates.add("  first type mismatch at position: " & $abs(err.firstMismatch))
-        if err.firstMismatch >= 0: candidates.add("\n  required type: ")
-        else: candidates.add("\n  unknown named parameter: " & $n[-err.firstMismatch][0])
-      var wanted, got: PType = nil
-      if err.firstMismatch < 0:
-        discard
-      elif err.firstMismatch < err.sym.typ.len:
-        wanted = err.sym.typ.sons[err.firstMismatch]
-        if cond: candidates.add typeToString(wanted)
-      else:
-        if cond: candidates.add "none"
-      if err.firstMismatch > 0 and err.firstMismatch < n.len:
-        if cond:
-          candidates.add "\n  but expression '"
-          candidates.add renderTree(n[err.firstMismatch])
+    let nArg = if err.firstMismatch.arg < n.len: n[err.firstMismatch.arg] else: nil
+    let nameParam = if err.firstMismatch.formal != nil: err.firstMismatch.formal.name.s else: ""
+    if n.len > 1:
+      candidates.add("  first type mismatch at position: " & $err.firstMismatch.arg)
+      # candidates.add "\n  reason: " & $err.firstMismatch.kind # for debugging
+      case err.firstMismatch.kind
+      of kUnknownNamedParam: candidates.add("\n  unknown named parameter: " & $nArg[0])
+      of kAlreadyGiven: candidates.add("\n  named param already provided: " & $nArg[0])
+      of kExtraArg: candidates.add("\n  extra argument given")
+      of kMissingParam: candidates.add("\n  missing parameter: " & nameParam)
+      of kTypeMismatch, kVarNeeded:
+        doAssert nArg != nil
+        var wanted = err.firstMismatch.formal.typ
+        doAssert err.firstMismatch.formal != nil
+        candidates.add("\n  required type for " & nameParam &  ": ")
+        candidates.add typeToString(wanted)
+        candidates.add "\n  but expression '"
+        if err.firstMismatch.kind == kVarNeeded:
+          candidates.add renderNotLValue(nArg)
+          candidates.add "' is immutable, not 'var'"
+        else:
+          candidates.add renderTree(nArg)
           candidates.add "' is of type: "
-        got = n[err.firstMismatch].typ
-        if cond: candidates.add typeToString(got)
-      if wanted != nil and got != nil:
-        effectProblem(wanted, got, candidates)
-      if cond: candidates.add "\n"
-    if err.unmatchedVarParam != 0 and err.unmatchedVarParam < n.len:
-      candidates.add("  for a 'var' type a variable needs to be passed, but '" &
-                      renderNotLValue(n[err.unmatchedVarParam]) &
-                      "' is immutable\n")
+          var got = nArg.typ
+          candidates.add typeToString(got)
+          doAssert wanted != nil
+          if got != nil: effectProblem(wanted, got, candidates)
+      of kUnknown: internalAssert(c.config, false)
+      candidates.add "\n"
     for diag in err.diagnostics:
       candidates.add(diag & "\n")
   if skipped > 0:
@@ -260,7 +260,7 @@ proc bracketNotFoundError(c: PContext; n: PNode) =
   while symx != nil:
     if symx.kind in routineKinds:
       errors.add(CandidateError(sym: symx,
-                                unmatchedVarParam: 0, firstMismatch: 0,
+                                firstMismatch: MismatchInfo(),
                                 diagnostics: @[],
                                 enabled: false))
     symx = nextOverloadIter(o, c, headSymbol)

compiler/semdata.nim

@@ -10,11 +10,8 @@
 ## This module contains the data structures for the semantic checking phase.
 
 import
-  strutils, intsets, options, lexer, ast, astalgo, trees, treetab,
-  wordrecg,
-  ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math,
-  magicsys, nversion, nimsets, parser, times, passes, vmdef,
-  modulegraphs, lineinfos
+  intsets, options, ast, astalgo, msgs, idents, renderer,
+  magicsys, vmdef, modulegraphs, lineinfos
 
 type
   TOptionEntry* = object      # entries to put on a stack for pragma parsing

compiler/semexprs.nim

@@ -2438,6 +2438,7 @@ proc semExportExcept(c: PContext, n: PNode): PNode =
       strTableAdd(c.module.tab, s)
       result.add newSymNode(s, n.info)
     s = nextIter(i, exported.tab)
+  markUsed(c.config, n.info, exported, c.graph.usageSym)
 
 proc semExport(c: PContext, n: PNode): PNode =
   result = newNodeI(nkExportStmt, n.info)

compiler/semfold.nim

@@ -11,8 +11,8 @@
 # and evaluation phase
 
 import
-  strutils, options, ast, astalgo, trees, treetab, nimsets,
-  nversion, platform, math, msgs, os, condsyms, idents, renderer, types,
+  strutils, options, ast, trees, nimsets,
+  platform, math, msgs, idents, renderer, types,
   commands, magicsys, modulegraphs, strtabs, lineinfos
 
 proc newIntNodeT*(intVal: BiggestInt, n: PNode; g: ModuleGraph): PNode =

compiler/semmacrosanity.nim

@@ -10,7 +10,7 @@
 ## Implements type sanity checking for ASTs resulting from macros. Lots of
 ## room for improvement here.
 
-import ast, astalgo, msgs, types, options
+import ast, msgs, types, options
 
 proc ithField(n: PNode, field: var int): PSym =
   result = nil

compiler/sempass2.nim

@@ -10,7 +10,7 @@
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
   wordrecg, strutils, options, guards, lineinfos, semfold, semdata,
-  modulegraphs, lowerings, sigmatch, tables
+  modulegraphs
 
 when not defined(leanCompiler):
   import writetracking
@@ -34,7 +34,7 @@ In the construct let/var x = expr() x's type is marked.
 
 In x = y the type of x is marked.
 
-For every sink parameter of type T T is marked. TODO!
+For every sink parameter of type T T is marked.
 
 For every call f() the return type of f() is marked.
 
@@ -977,6 +977,14 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
   var t: TEffects
   initEffects(g, effects, s, t, c)
   track(t, body)
+
+  if s.kind != skMacro:
+    let params = s.typ.n
+    for i in 1 ..< params.len:
+      let param = params[i].sym
+      if isSinkTypeForParam(param.typ):
+        createTypeBoundOps(t.graph, t.c, param.typ, param.info)
+
   if not isEmptyType(s.typ.sons[0]) and
       ({tfNeedsInit, tfNotNil} * s.typ.sons[0].flags != {} or
       s.typ.sons[0].skipTypes(abstractInst).kind == tyVar) and

compiler/semtypes.nim

@@ -1812,14 +1812,11 @@ proc setMagicType(conf: ConfigRef; m: PSym, kind: TTypeKind, size: int) =
   # long double size can be 8, 10, 12, 16 bytes depending on platform & compiler
   if conf.target.targetCPU == cpuI386 and size == 8:
     #on Linux/BSD i386, double are aligned to 4bytes (except with -malign-double)
-    if kind in {tyFloat64, tyFloat} and
-        conf.target.targetOS in {osLinux, osAndroid, osNetbsd, osFreebsd, osOpenbsd, osDragonfly}:
-      m.typ.align = 4
-    # on i386, all known compiler, 64bits ints are aligned to 4bytes (except with -malign-double)
-    elif kind in {tyInt, tyUInt, tyInt64, tyUInt64}:
-      m.typ.align = 4
-  else:
-    discard
+    if conf.target.targetOS != osWindows:
+      if kind in {tyFloat64, tyFloat, tyInt, tyUInt, tyInt64, tyUInt64}:
+        # on i386 for all known POSIX systems, 64bits ints are aligned
+        # to 4bytes (except with -malign-double)
+        m.typ.align = 4
 
 proc setMagicIntegral(conf: ConfigRef; m: PSym, kind: TTypeKind, size: int) =
   setMagicType(conf, m, kind, size)

compiler/sighashes.nim

@@ -11,9 +11,7 @@
 
 import ast, tables, ropes, md5, modulegraphs
 from hashes import Hash
-from astalgo import debug
 import types
-from strutils import startsWith, contains
 
 proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
 proc `&=`(c: var MD5Context, ch: char) = md5Update(c, unsafeAddr ch, 1)

compiler/sigmatch.nim

@@ -12,19 +12,29 @@
 
 import
   intsets, ast, astalgo, semdata, types, msgs, renderer, lookups, semtypinst,
-  magicsys, condsyms, idents, lexer, options, parampatterns, strutils, trees,
+  magicsys, idents, lexer, options, parampatterns, strutils, trees,
   linter, lineinfos, lowerings, modulegraphs
 
 when (defined(booting) or defined(nimsuggest)) and not defined(leanCompiler):
   import docgen
 
 type
+  MismatchKind* = enum
+    kUnknown, kAlreadyGiven, kUnknownNamedParam, kTypeMismatch, kVarNeeded,
+    kMissingParam, kExtraArg
+
+  MismatchInfo* = object
+    kind*: MismatchKind # reason for mismatch
+    arg*: int           # position of provided arguments that mismatches
+    formal*: PSym       # parameter that mismatches against provided argument
+                        # its position can differ from `arg` because of varargs
+
   TCandidateState* = enum
     csEmpty, csMatch, csNoMatch
 
   CandidateError* = object
     sym*: PSym
-    unmatchedVarParam*, firstMismatch*: int
+    firstMismatch*: MismatchInfo
     diagnostics*: seq[string]
     enabled*: bool
 
@@ -56,7 +66,6 @@ type
                              # a distrinct type
     typedescMatched*: bool
     isNoCall*: bool          # misused for generic type instantiations C[T]
-    mutabilityProblem*: uint8 # tyVar mismatch
     inferredTypes: seq[PType] # inferred types during the current signature
                               # matching. they will be reset if the matching
                               # is not successful. may replace the bindings
@@ -70,8 +79,7 @@ type
                               # triggered with an idetools command in the
                               # future.
     inheritancePenalty: int   # to prefer closest father object type
-    firstMismatch*: int       # position of the first type mismatch for
-                              # better error messages
+    firstMismatch*: MismatchInfo # mismatch info for better error messages
     diagnosticsEnabled*: bool
 
   TTypeRelFlag* = enum
@@ -112,6 +120,7 @@ proc initCandidateAux(ctx: PContext,
   c.intConvMatches = 0
   c.genericMatches = 0
   c.state = csEmpty
+  c.firstMismatch = MismatchInfo()
   c.callee = callee
   c.call = nil
   c.baseTypeMatch = false
@@ -331,8 +340,6 @@ proc typeRel*(c: var TCandidate, f, aOrig: PType,
 
 proc concreteType(c: TCandidate, t: PType; f: PType = nil): PType =
   case t.kind
-  of tyNil:
-    result = nil              # what should it be?
   of tyTypeDesc:
     if c.isNoCall: result = t
     else: result = nil
@@ -2108,7 +2115,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
       # lift do blocks without params to lambdas
       let p = c.graph
       let lifted = c.semExpr(c, newProcNode(nkDo, argOrig.info, body = argOrig,
-          params = p.emptyNode, name = p.emptyNode, pattern = p.emptyNode,
+          params = nkFormalParams.newTree(p.emptyNode), name = p.emptyNode, pattern = p.emptyNode,
           genericParams = p.emptyNode, pragmas = p.emptyNode, exceptions = p.emptyNode), {})
       if f.kind == tyBuiltInTypeClass:
         inc m.genericMatches
@@ -2280,6 +2287,17 @@ template isVarargsUntyped(x): untyped =
 
 proc matchesAux(c: PContext, n, nOrig: PNode,
                 m: var TCandidate, marker: var IntSet) =
+  var
+    a = 1 # iterates over the actual given arguments
+    f = if m.callee.kind != tyGenericBody: 1
+        else: 0 # iterates over formal parameters
+    arg: PNode # current prepared argument
+    formal: PSym # current routine parameter
+
+  defer:
+    m.firstMismatch.arg = a
+    m.firstMismatch.formal = formal
+
   template checkConstraint(n: untyped) {.dirty.} =
     if not formal.constraint.isNil:
       if matchNodeKinds(formal.constraint, n):
@@ -2294,28 +2312,21 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
       if argConverter.kind == nkHiddenCallConv:
         if argConverter.typ.kind != tyVar:
           m.state = csNoMatch
-          m.mutabilityProblem = uint8(f-1)
+          m.firstMismatch.kind = kVarNeeded
           return
       elif not n.isLValue:
         m.state = csNoMatch
-        m.mutabilityProblem = uint8(f-1)
+        m.firstMismatch.kind = kVarNeeded
         return
 
-  var
-    # iterates over formal parameters
-    f = if m.callee.kind != tyGenericBody: 1
-        else: 0
-    # iterates over the actual given arguments
-    a = 1
-    arg: PNode # current prepared argument
-
   m.state = csMatch # until proven otherwise
+  m.firstMismatch = MismatchInfo()
   m.call = newNodeI(n.kind, n.info)
   m.call.typ = base(m.callee) # may be nil
   var formalLen = m.callee.n.len
-  addSon(m.call, copyTree(n.sons[0]))
+  addSon(m.call, n.sons[0])
   var container: PNode = nil # constructed container
-  var formal: PSym = if formalLen > 1: m.callee.n.sons[1].sym else: nil
+  formal = if formalLen > 1: m.callee.n.sons[1].sym else: nil
 
   while a < n.len:
     if a >= formalLen-1 and f < formalLen and m.callee.n[f].typ.isVarargsUntyped:
@@ -2336,20 +2347,20 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
         addSon(container, n.sons[a])
     elif n.sons[a].kind == nkExprEqExpr:
       # named param
+      m.firstMismatch.kind = kUnknownNamedParam
       # check if m.callee has such a param:
       prepareNamedParam(n.sons[a], c)
       if n.sons[a].sons[0].kind != nkIdent:
         localError(c.config, n.sons[a].info, "named parameter has to be an identifier")
         m.state = csNoMatch
-        m.firstMismatch = -a
         return
       formal = getSymFromList(m.callee.n, n.sons[a].sons[0].ident, 1)
       if formal == nil:
         # no error message!
         m.state = csNoMatch
-        m.firstMismatch = -a
         return
       if containsOrIncl(marker, formal.position):
+        m.firstMismatch.kind = kAlreadyGiven
         # already in namedParams, so no match
         # we used to produce 'errCannotBindXTwice' here but see
         # bug #3836 of why that is not sound (other overload with
@@ -2363,9 +2374,9 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
       n.sons[a].typ = n.sons[a].sons[1].typ
       arg = paramTypesMatch(m, formal.typ, n.sons[a].typ,
                                 n.sons[a].sons[1], n.sons[a].sons[1])
+      m.firstMismatch.kind = kTypeMismatch
       if arg == nil:
         m.state = csNoMatch
-        m.firstMismatch = a
         return
       checkConstraint(n.sons[a].sons[1])
       if m.baseTypeMatch:
@@ -2392,6 +2403,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
           else:
             addSon(m.call, copyTree(n.sons[a]))
         elif formal != nil and formal.typ.kind == tyVarargs:
+          m.firstMismatch.kind = kTypeMismatch
           # beware of the side-effects in 'prepareOperand'! So only do it for
           # varargs matching. See tests/metatype/tstatic_overloading.
           m.baseTypeMatch = false
@@ -2408,6 +2420,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
             m.state = csNoMatch
             return
         else:
+          m.firstMismatch.kind = kExtraArg
           m.state = csNoMatch
           return
       else:
@@ -2415,7 +2428,9 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
           internalError(c.config, n.sons[a].info, "matches")
           return
         formal = m.callee.n.sons[f].sym
+        m.firstMismatch.kind = kTypeMismatch
         if containsOrIncl(marker, formal.position) and container.isNil:
+          m.firstMismatch.kind = kAlreadyGiven
           # already in namedParams: (see above remark)
           when false: localError(n.sons[a].info, errCannotBindXTwice, formal.name.s)
           m.state = csNoMatch
@@ -2436,7 +2451,6 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
                                     n.sons[a], nOrig.sons[a])
           if arg == nil:
             m.state = csNoMatch
-            m.firstMismatch = f
             return
           if m.baseTypeMatch:
             assert formal.typ.kind == tyVarargs
@@ -2510,7 +2524,8 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) =
         else:
           # no default value
           m.state = csNoMatch
-          m.firstMismatch = f
+          m.firstMismatch.kind = kMissingParam
+          m.firstMismatch.formal = formal
           break
       else:
         if formal.ast.kind == nkEmpty:

compiler/sizealignoffsetimpl.nim

@@ -169,7 +169,7 @@ proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode,
       align = n.sym.typ.align.int
 
     result.align = align
-    if initialOffset == szUnknownSize or size == szUnknownSize:
+    if initialOffset == szUnknownSize or size == szUnknownSize or align == szUnknownSize:
       n.sym.offset = szUnknownSize
       result.offset = szUnknownSize
     else:

compiler/spawn.nim

@@ -1,428 +1,428 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements threadpool's ``spawn``.
-
-import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
-  lineinfos, lowerings
-from trees import getMagic
-
-proc callProc(a: PNode): PNode =
-  result = newNodeI(nkCall, a.info)
-  result.add a
-  result.typ = a.typ.sons[0]
-
-# we have 4 cases to consider:
-# - a void proc --> nothing to do
-# - a proc returning GC'ed memory --> requires a flowVar
-# - a proc returning non GC'ed memory --> pass as hidden 'var' parameter
-# - not in a parallel environment --> requires a flowVar for memory safety
-type
-  TSpawnResult* = enum
-    srVoid, srFlowVar, srByVar
-  TFlowVarKind = enum
-    fvInvalid # invalid type T for 'FlowVar[T]'
-    fvGC      # FlowVar of a GC'ed type
-    fvBlob    # FlowVar of a blob type
-
-proc spawnResult*(t: PType; inParallel: bool): TSpawnResult =
-  if t.isEmptyType: srVoid
-  elif inParallel and not containsGarbageCollectedRef(t): srByVar
-  else: srFlowVar
-
-proc flowVarKind(t: PType): TFlowVarKind =
-  if t.skipTypes(abstractInst).kind in {tyRef, tyString, tySequence}: fvGC
-  elif containsGarbageCollectedRef(t): fvInvalid
-  else: fvBlob
-
-proc typeNeedsNoDeepCopy(t: PType): bool =
-  var t = t.skipTypes(abstractInst)
-  # for the tconvexhull example (and others) we're a bit lax here and pretend
-  # seqs and strings are *by value* only and 'shallow' doesn't exist!
-  if t.kind == tyString: return true
-  # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var'
-  # for the stricter check and likewise we can skip 'seq' for a less
-  # strict check:
-  if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon
-  result = not containsGarbageCollectedRef(t)
-
-proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; owner: PSym; typ: PType;
-                 v: PNode; useShallowCopy=false): PSym =
-  result = newSym(skTemp, getIdent(g.cache, genPrefix), owner, varSection.info,
-                  owner.options)
-  result.typ = typ
-  incl(result.flags, sfFromGeneric)
-
-  var vpart = newNodeI(nkIdentDefs, varSection.info, 3)
-  vpart.sons[0] = newSymNode(result)
-  vpart.sons[1] = newNodeI(nkEmpty, varSection.info)
-  vpart.sons[2] = if varInit.isNil: v else: vpart[1]
-  varSection.add vpart
-  if varInit != nil:
-    if useShallowCopy and typeNeedsNoDeepCopy(typ):
-      varInit.add newFastAsgnStmt(newSymNode(result), v)
-    else:
-      let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
-      deepCopyCall.sons[0] = newSymNode(getSysMagic(g, varSection.info, "deepCopy", mDeepCopy))
-      deepCopyCall.sons[1] = newSymNode(result)
-      deepCopyCall.sons[2] = v
-      varInit.add deepCopyCall
-
-discard """
-We generate roughly this:
-
-proc f_wrapper(thread, args) =
-  barrierEnter(args.barrier)  # for parallel statement
-  var a = args.a # thread transfer; deepCopy or shallowCopy or no copy
-                 # depending on whether we're in a 'parallel' statement
-  var b = args.b
-  var fv = args.fv
-
-  fv.owner = thread # optional
-  nimArgsPassingDone() # signal parent that the work is done
-  #
-  args.fv.blob = f(a, b, ...)
-  nimFlowVarSignal(args.fv)
-
-  # - or -
-  f(a, b, ...)
-  barrierLeave(args.barrier)  # for parallel statement
-
-stmtList:
-  var scratchObj
-  scratchObj.a = a
-  scratchObj.b = b
-
-  nimSpawn(f_wrapper, addr scratchObj)
-  scratchObj.fv # optional
-
-"""
-
-proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
-                       varSection, varInit, call, barrier, fv: PNode;
-                       spawnKind: TSpawnResult): PSym =
-  var body = newNodeI(nkStmtList, f.info)
-  body.flags.incl nfTransf # do not transform further
-
-  var threadLocalBarrier: PSym
-  if barrier != nil:
-    var varSection2 = newNodeI(nkVarSection, barrier.info)
-    threadLocalBarrier = addLocalVar(g, varSection2, nil, argsParam.owner,
-                                     barrier.typ, barrier)
-    body.add varSection2
-    body.add callCodegenProc(g, "barrierEnter", threadLocalBarrier.info,
-      threadLocalBarrier.newSymNode)
-  var threadLocalProm: PSym
-  if spawnKind == srByVar:
-    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
-  elif fv != nil:
-    internalAssert g.config, fv.typ.kind == tyGenericInst
-    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
-  body.add varSection
-  body.add varInit
-  if fv != nil and spawnKind != srByVar:
-    # generate:
-    #   fv.owner = threadParam
-    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
-      "owner", fv.info, g.cache), threadParam.newSymNode)
-
-  body.add callCodegenProc(g, "nimArgsPassingDone", threadParam.info,
-    threadParam.newSymNode)
-  if spawnKind == srByVar:
-    body.add newAsgnStmt(genDeref(threadLocalProm.newSymNode), call)
-  elif fv != nil:
-    let fk = fv.typ.sons[1].flowVarKind
-    if fk == fvInvalid:
-      localError(g.config, f.info, "cannot create a flowVar of type: " &
-        typeToString(fv.typ.sons[1]))
-    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
-      if fk == fvGC: "data" else: "blob", fv.info, g.cache), call)
-    if fk == fvGC:
-      let incRefCall = newNodeI(nkCall, fv.info, 2)
-      incRefCall.sons[0] = newSymNode(getSysMagic(g, fv.info, "GCref", mGCref))
-      incRefCall.sons[1] = indirectAccess(threadLocalProm.newSymNode,
-                                          "data", fv.info, g.cache)
-      body.add incRefCall
-    if barrier == nil:
-      # by now 'fv' is shared and thus might have beeen overwritten! we need
-      # to use the thread-local view instead:
-      body.add callCodegenProc(g, "nimFlowVarSignal", threadLocalProm.info,
-        threadLocalProm.newSymNode)
-  else:
-    body.add call
-  if barrier != nil:
-    body.add callCodegenProc(g, "barrierLeave", threadLocalBarrier.info,
-      threadLocalBarrier.newSymNode)
-
-  var params = newNodeI(nkFormalParams, f.info)
-  params.add newNodeI(nkEmpty, f.info)
-  params.add threadParam.newSymNode
-  params.add argsParam.newSymNode
-
-  var t = newType(tyProc, threadParam.owner)
-  t.rawAddSon nil
-  t.rawAddSon threadParam.typ
-  t.rawAddSon argsParam.typ
-  t.n = newNodeI(nkFormalParams, f.info)
-  t.n.add newNodeI(nkEffectList, f.info)
-  t.n.add threadParam.newSymNode
-  t.n.add argsParam.newSymNode
-
-  let name = (if f.kind == nkSym: f.sym.name.s else: genPrefix) & "Wrapper"
-  result = newSym(skProc, getIdent(g.cache, name), argsParam.owner, f.info,
-                  argsParam.options)
-  let emptyNode = newNodeI(nkEmpty, f.info)
-  result.ast = newProcNode(nkProcDef, f.info, body = body,
-      params = params, name = newSymNode(result), pattern = emptyNode,
-      genericParams = emptyNode, pragmas = emptyNode,
-      exceptions = emptyNode)
-  result.typ = t
-
-proc createCastExpr(argsParam: PSym; objType: PType): PNode =
-  result = newNodeI(nkCast, argsParam.info)
-  result.add newNodeI(nkEmpty, argsParam.info)
-  result.add newSymNode(argsParam)
-  result.typ = newType(tyPtr, objType.owner)
-  result.typ.rawAddSon(objType)
-
-proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym,
-                             castExpr, call,
-                             varSection, varInit, result: PNode) =
-  let formals = n[0].typ.n
-  let tmpName = getIdent(g.cache, genPrefix)
-  for i in 1 ..< n.len:
-    # we pick n's type here, which hopefully is 'tyArray' and not
-    # 'tyOpenArray':
-    var argType = n[i].typ.skipTypes(abstractInst)
-    if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}:
-      localError(g.config, n[i].info, "'spawn'ed function cannot have a 'var' parameter")
-    #elif containsTyRef(argType):
-    #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
-
-    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
-    field.typ = argType
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
-
-    let temp = addLocalVar(g, varSection, varInit, objType.owner, argType,
-                           indirectAccess(castExpr, field, n.info))
-    call.add(newSymNode(temp))
-
-proc getRoot*(n: PNode): PSym =
-  ## ``getRoot`` takes a *path* ``n``. A path is an lvalue expression
-  ## like ``obj.x[i].y``. The *root* of a path is the symbol that can be
-  ## determined as the owner; ``obj`` in the example.
-  case n.kind
-  of nkSym:
-    if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar}:
-      result = n.sym
-  of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
-      nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
-    result = getRoot(n.sons[0])
-  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-    result = getRoot(n.sons[1])
-  of nkCallKinds:
-    if getMagic(n) == mSlice: result = getRoot(n.sons[1])
-  else: discard
-
-proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym;
-                             castExpr, call,
-                             varSection, varInit, result: PNode) =
-  let formals = n[0].typ.n
-  let tmpName = getIdent(g.cache, genPrefix)
-  # we need to copy the foreign scratch object fields into local variables
-  # for correctness: These are called 'threadLocal' here.
-  for i in 1 ..< n.len:
-    let n = n[i]
-    let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ,
-                            abstractInst)
-    #if containsTyRef(argType):
-    #  localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
-
-    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
-
-    if argType.kind in {tyVarargs, tyOpenArray}:
-      # important special case: we always create a zero-copy slice:
-      let slice = newNodeI(nkCall, n.info, 4)
-      slice.typ = n.typ
-      slice.sons[0] = newSymNode(createMagic(g, "slice", mSlice))
-      slice.sons[0].typ = getSysType(g, n.info, tyInt) # fake type
-      var fieldB = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
-      fieldB.typ = getSysType(g, n.info, tyInt)
-      objType.addField(fieldB, g.cache)
-
-      if getMagic(n) == mSlice:
-        let a = genAddrOf(n[1])
-        field.typ = a.typ
-        objType.addField(field, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-
-        var fieldA = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
-        fieldA.typ = getSysType(g, n.info, tyInt)
-        objType.addField(fieldA, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3])
-
-        let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldA.typ,
-                                      indirectAccess(castExpr, fieldA, n.info),
-                                      useShallowCopy=true)
-        slice.sons[2] = threadLocal.newSymNode
-      else:
-        let a = genAddrOf(n)
-        field.typ = a.typ
-        objType.addField(field, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n))
-
-        slice.sons[2] = newIntLit(g, n.info, 0)
-      # the array itself does not need to go through a thread local variable:
-      slice.sons[1] = genDeref(indirectAccess(castExpr, field, n.info))
-
-      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldB.typ,
-                                    indirectAccess(castExpr, fieldB, n.info),
-                                    useShallowCopy=true)
-      slice.sons[3] = threadLocal.newSymNode
-      call.add slice
-    elif (let size = computeSize(g.config, argType); size < 0 or size > 16) and
-        n.getRoot != nil:
-      # it is more efficient to pass a pointer instead:
-      let a = genAddrOf(n)
-      field.typ = a.typ
-      objType.addField(field, g.cache)
-      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, field.typ,
-                                    indirectAccess(castExpr, field, n.info),
-                                    useShallowCopy=true)
-      call.add(genDeref(threadLocal.newSymNode))
-    else:
-      # boring case
-      field.typ = argType
-      objType.addField(field, g.cache)
-      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n)
-      let threadLocal = addLocalVar(g, varSection, varInit,
-                                    objType.owner, field.typ,
-                                    indirectAccess(castExpr, field, n.info),
-                                    useShallowCopy=true)
-      call.add(threadLocal.newSymNode)
-
-proc wrapProcForSpawn*(g: ModuleGraph; owner: PSym; spawnExpr: PNode; retType: PType;
-                       barrier, dest: PNode = nil): PNode =
-  # if 'barrier' != nil, then it is in a 'parallel' section and we
-  # generate quite different code
-  let n = spawnExpr[^2]
-  let spawnKind = spawnResult(retType, barrier!=nil)
-  case spawnKind
-  of srVoid:
-    internalAssert g.config, dest == nil
-    result = newNodeI(nkStmtList, n.info)
-  of srFlowVar:
-    internalAssert g.config, dest == nil
-    result = newNodeIT(nkStmtListExpr, n.info, retType)
-  of srByVar:
-    if dest == nil: localError(g.config, n.info, "'spawn' must not be discarded")
-    result = newNodeI(nkStmtList, n.info)
-
-  if n.kind notin nkCallKinds:
-    localError(g.config, n.info, "'spawn' takes a call expression")
-    return
-  if optThreadAnalysis in g.config.globalOptions:
-    if {tfThread, tfNoSideEffect} * n[0].typ.flags == {}:
-      localError(g.config, n.info, "'spawn' takes a GC safe call expression")
-  var
-    threadParam = newSym(skParam, getIdent(g.cache, "thread"), owner, n.info, g.config.options)
-    argsParam = newSym(skParam, getIdent(g.cache, "args"), owner, n.info, g.config.options)
-  block:
-    let ptrType = getSysType(g, n.info, tyPointer)
-    threadParam.typ = ptrType
-    argsParam.typ = ptrType
-    argsParam.position = 1
-
-  var objType = createObj(g, owner, n.info)
-  incl(objType.flags, tfFinal)
-  let castExpr = createCastExpr(argsParam, objType)
-
-  var scratchObj = newSym(skVar, getIdent(g.cache, "scratch"), owner, n.info, g.config.options)
-  block:
-    scratchObj.typ = objType
-    incl(scratchObj.flags, sfFromGeneric)
-    var varSectionB = newNodeI(nkVarSection, n.info)
-    varSectionB.addVar(scratchObj.newSymNode)
-    result.add varSectionB
-
-  var call = newNodeIT(nkCall, n.info, n.typ)
-  var fn = n.sons[0]
-  # templates and macros are in fact valid here due to the nature of
-  # the transformation:
-  if fn.kind == nkClosure or (fn.typ != nil and fn.typ.callConv == ccClosure):
-    localError(g.config, n.info, "closure in spawn environment is not allowed")
-  if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro,
-                                               skFunc, skMethod, skConverter}):
-    # for indirect calls we pass the function pointer in the scratchObj
-    var argType = n[0].typ.skipTypes(abstractInst)
-    var field = newSym(skField, getIdent(g.cache, "fn"), owner, n.info, g.config.options)
-    field.typ = argType
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
-    fn = indirectAccess(castExpr, field, n.info)
-  elif fn.kind == nkSym and fn.sym.kind == skIterator:
-    localError(g.config, n.info, "iterator in spawn environment is not allowed")
-  elif fn.typ.callConv == ccClosure:
-    localError(g.config, n.info, "closure in spawn environment is not allowed")
-
-  call.add(fn)
-  var varSection = newNodeI(nkVarSection, n.info)
-  var varInit = newNodeI(nkStmtList, n.info)
-  if barrier.isNil:
-    setupArgsForConcurrency(g, n, objType, scratchObj, castExpr, call,
-                            varSection, varInit, result)
-  else:
-    setupArgsForParallelism(g, n, objType, scratchObj, castExpr, call,
-                            varSection, varInit, result)
-
-  var barrierAsExpr: PNode = nil
-  if barrier != nil:
-    let typ = newType(tyPtr, owner)
-    typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ)
-    var field = newSym(skField, getIdent(g.cache, "barrier"), owner, n.info, g.config.options)
-    field.typ = typ
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier)
-    barrierAsExpr = indirectAccess(castExpr, field, n.info)
-
-  var fvField, fvAsExpr: PNode = nil
-  if spawnKind == srFlowVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
-    field.typ = retType
-    objType.addField(field, g.cache)
-    fvField = newDotExpr(scratchObj, field)
-    fvAsExpr = indirectAccess(castExpr, field, n.info)
-    # create flowVar:
-    result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1]))
-    if barrier == nil:
-      result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField.info,
-        fvField)
-
-  elif spawnKind == srByVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
-    field.typ = newType(tyPtr, objType.owner)
-    field.typ.rawAddSon(retType)
-    objType.addField(field, g.cache)
-    fvAsExpr = indirectAccess(castExpr, field, n.info)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest))
-
-  let wrapper = createWrapperProc(g, fn, threadParam, argsParam,
-                                  varSection, varInit, call,
-                                  barrierAsExpr, fvAsExpr, spawnKind)
-  result.add callCodegenProc(g, "nimSpawn" & $spawnExpr.len, wrapper.info,
-    wrapper.newSymNode, genAddrOf(scratchObj.newSymNode), nil, spawnExpr)
-
-  if spawnKind == srFlowVar: result.add fvField
-
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements threadpool's ``spawn``.
+
+import ast, types, idents, magicsys, msgs, options, modulegraphs,
+  lowerings
+from trees import getMagic
+
+proc callProc(a: PNode): PNode =
+  result = newNodeI(nkCall, a.info)
+  result.add a
+  result.typ = a.typ.sons[0]
+
+# we have 4 cases to consider:
+# - a void proc --> nothing to do
+# - a proc returning GC'ed memory --> requires a flowVar
+# - a proc returning non GC'ed memory --> pass as hidden 'var' parameter
+# - not in a parallel environment --> requires a flowVar for memory safety
+type
+  TSpawnResult* = enum
+    srVoid, srFlowVar, srByVar
+  TFlowVarKind = enum
+    fvInvalid # invalid type T for 'FlowVar[T]'
+    fvGC      # FlowVar of a GC'ed type
+    fvBlob    # FlowVar of a blob type
+
+proc spawnResult*(t: PType; inParallel: bool): TSpawnResult =
+  if t.isEmptyType: srVoid
+  elif inParallel and not containsGarbageCollectedRef(t): srByVar
+  else: srFlowVar
+
+proc flowVarKind(t: PType): TFlowVarKind =
+  if t.skipTypes(abstractInst).kind in {tyRef, tyString, tySequence}: fvGC
+  elif containsGarbageCollectedRef(t): fvInvalid
+  else: fvBlob
+
+proc typeNeedsNoDeepCopy(t: PType): bool =
+  var t = t.skipTypes(abstractInst)
+  # for the tconvexhull example (and others) we're a bit lax here and pretend
+  # seqs and strings are *by value* only and 'shallow' doesn't exist!
+  if t.kind == tyString: return true
+  # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var'
+  # for the stricter check and likewise we can skip 'seq' for a less
+  # strict check:
+  if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon
+  result = not containsGarbageCollectedRef(t)
+
+proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; owner: PSym; typ: PType;
+                 v: PNode; useShallowCopy=false): PSym =
+  result = newSym(skTemp, getIdent(g.cache, genPrefix), owner, varSection.info,
+                  owner.options)
+  result.typ = typ
+  incl(result.flags, sfFromGeneric)
+
+  var vpart = newNodeI(nkIdentDefs, varSection.info, 3)
+  vpart.sons[0] = newSymNode(result)
+  vpart.sons[1] = newNodeI(nkEmpty, varSection.info)
+  vpart.sons[2] = if varInit.isNil: v else: vpart[1]
+  varSection.add vpart
+  if varInit != nil:
+    if useShallowCopy and typeNeedsNoDeepCopy(typ):
+      varInit.add newFastAsgnStmt(newSymNode(result), v)
+    else:
+      let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
+      deepCopyCall.sons[0] = newSymNode(getSysMagic(g, varSection.info, "deepCopy", mDeepCopy))
+      deepCopyCall.sons[1] = newSymNode(result)
+      deepCopyCall.sons[2] = v
+      varInit.add deepCopyCall
+
+discard """
+We generate roughly this:
+
+proc f_wrapper(thread, args) =
+  barrierEnter(args.barrier)  # for parallel statement
+  var a = args.a # thread transfer; deepCopy or shallowCopy or no copy
+                 # depending on whether we're in a 'parallel' statement
+  var b = args.b
+  var fv = args.fv
+
+  fv.owner = thread # optional
+  nimArgsPassingDone() # signal parent that the work is done
+  #
+  args.fv.blob = f(a, b, ...)
+  nimFlowVarSignal(args.fv)
+
+  # - or -
+  f(a, b, ...)
+  barrierLeave(args.barrier)  # for parallel statement
+
+stmtList:
+  var scratchObj
+  scratchObj.a = a
+  scratchObj.b = b
+
+  nimSpawn(f_wrapper, addr scratchObj)
+  scratchObj.fv # optional
+
+"""
+
+proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
+                       varSection, varInit, call, barrier, fv: PNode;
+                       spawnKind: TSpawnResult): PSym =
+  var body = newNodeI(nkStmtList, f.info)
+  body.flags.incl nfTransf # do not transform further
+
+  var threadLocalBarrier: PSym
+  if barrier != nil:
+    var varSection2 = newNodeI(nkVarSection, barrier.info)
+    threadLocalBarrier = addLocalVar(g, varSection2, nil, argsParam.owner,
+                                     barrier.typ, barrier)
+    body.add varSection2
+    body.add callCodegenProc(g, "barrierEnter", threadLocalBarrier.info,
+      threadLocalBarrier.newSymNode)
+  var threadLocalProm: PSym
+  if spawnKind == srByVar:
+    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
+  elif fv != nil:
+    internalAssert g.config, fv.typ.kind == tyGenericInst
+    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
+  body.add varSection
+  body.add varInit
+  if fv != nil and spawnKind != srByVar:
+    # generate:
+    #   fv.owner = threadParam
+    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
+      "owner", fv.info, g.cache), threadParam.newSymNode)
+
+  body.add callCodegenProc(g, "nimArgsPassingDone", threadParam.info,
+    threadParam.newSymNode)
+  if spawnKind == srByVar:
+    body.add newAsgnStmt(genDeref(threadLocalProm.newSymNode), call)
+  elif fv != nil:
+    let fk = fv.typ.sons[1].flowVarKind
+    if fk == fvInvalid:
+      localError(g.config, f.info, "cannot create a flowVar of type: " &
+        typeToString(fv.typ.sons[1]))
+    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
+      if fk == fvGC: "data" else: "blob", fv.info, g.cache), call)
+    if fk == fvGC:
+      let incRefCall = newNodeI(nkCall, fv.info, 2)
+      incRefCall.sons[0] = newSymNode(getSysMagic(g, fv.info, "GCref", mGCref))
+      incRefCall.sons[1] = indirectAccess(threadLocalProm.newSymNode,
+                                          "data", fv.info, g.cache)
+      body.add incRefCall
+    if barrier == nil:
+      # by now 'fv' is shared and thus might have beeen overwritten! we need
+      # to use the thread-local view instead:
+      body.add callCodegenProc(g, "nimFlowVarSignal", threadLocalProm.info,
+        threadLocalProm.newSymNode)
+  else:
+    body.add call
+  if barrier != nil:
+    body.add callCodegenProc(g, "barrierLeave", threadLocalBarrier.info,
+      threadLocalBarrier.newSymNode)
+
+  var params = newNodeI(nkFormalParams, f.info)
+  params.add newNodeI(nkEmpty, f.info)
+  params.add threadParam.newSymNode
+  params.add argsParam.newSymNode
+
+  var t = newType(tyProc, threadParam.owner)
+  t.rawAddSon nil
+  t.rawAddSon threadParam.typ
+  t.rawAddSon argsParam.typ
+  t.n = newNodeI(nkFormalParams, f.info)
+  t.n.add newNodeI(nkEffectList, f.info)
+  t.n.add threadParam.newSymNode
+  t.n.add argsParam.newSymNode
+
+  let name = (if f.kind == nkSym: f.sym.name.s else: genPrefix) & "Wrapper"
+  result = newSym(skProc, getIdent(g.cache, name), argsParam.owner, f.info,
+                  argsParam.options)
+  let emptyNode = newNodeI(nkEmpty, f.info)
+  result.ast = newProcNode(nkProcDef, f.info, body = body,
+      params = params, name = newSymNode(result), pattern = emptyNode,
+      genericParams = emptyNode, pragmas = emptyNode,
+      exceptions = emptyNode)
+  result.typ = t
+
+proc createCastExpr(argsParam: PSym; objType: PType): PNode =
+  result = newNodeI(nkCast, argsParam.info)
+  result.add newNodeI(nkEmpty, argsParam.info)
+  result.add newSymNode(argsParam)
+  result.typ = newType(tyPtr, objType.owner)
+  result.typ.rawAddSon(objType)
+
+proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym,
+                             castExpr, call,
+                             varSection, varInit, result: PNode) =
+  let formals = n[0].typ.n
+  let tmpName = getIdent(g.cache, genPrefix)
+  for i in 1 ..< n.len:
+    # we pick n's type here, which hopefully is 'tyArray' and not
+    # 'tyOpenArray':
+    var argType = n[i].typ.skipTypes(abstractInst)
+    if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}:
+      localError(g.config, n[i].info, "'spawn'ed function cannot have a 'var' parameter")
+    #elif containsTyRef(argType):
+    #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
+
+    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
+    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
+    field.typ = argType
+    objType.addField(field, g.cache)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
+
+    let temp = addLocalVar(g, varSection, varInit, objType.owner, argType,
+                           indirectAccess(castExpr, field, n.info))
+    call.add(newSymNode(temp))
+
+proc getRoot*(n: PNode): PSym =
+  ## ``getRoot`` takes a *path* ``n``. A path is an lvalue expression
+  ## like ``obj.x[i].y``. The *root* of a path is the symbol that can be
+  ## determined as the owner; ``obj`` in the example.
+  case n.kind
+  of nkSym:
+    if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar}:
+      result = n.sym
+  of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
+      nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
+    result = getRoot(n.sons[0])
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    result = getRoot(n.sons[1])
+  of nkCallKinds:
+    if getMagic(n) == mSlice: result = getRoot(n.sons[1])
+  else: discard
+
+proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym;
+                             castExpr, call,
+                             varSection, varInit, result: PNode) =
+  let formals = n[0].typ.n
+  let tmpName = getIdent(g.cache, genPrefix)
+  # we need to copy the foreign scratch object fields into local variables
+  # for correctness: These are called 'threadLocal' here.
+  for i in 1 ..< n.len:
+    let n = n[i]
+    let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ,
+                            abstractInst)
+    #if containsTyRef(argType):
+    #  localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
+
+    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
+    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
+
+    if argType.kind in {tyVarargs, tyOpenArray}:
+      # important special case: we always create a zero-copy slice:
+      let slice = newNodeI(nkCall, n.info, 4)
+      slice.typ = n.typ
+      slice.sons[0] = newSymNode(createMagic(g, "slice", mSlice))
+      slice.sons[0].typ = getSysType(g, n.info, tyInt) # fake type
+      var fieldB = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
+      fieldB.typ = getSysType(g, n.info, tyInt)
+      objType.addField(fieldB, g.cache)
+
+      if getMagic(n) == mSlice:
+        let a = genAddrOf(n[1])
+        field.typ = a.typ
+        objType.addField(field, g.cache)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
+
+        var fieldA = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
+        fieldA.typ = getSysType(g, n.info, tyInt)
+        objType.addField(fieldA, g.cache)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3])
+
+        let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldA.typ,
+                                      indirectAccess(castExpr, fieldA, n.info),
+                                      useShallowCopy=true)
+        slice.sons[2] = threadLocal.newSymNode
+      else:
+        let a = genAddrOf(n)
+        field.typ = a.typ
+        objType.addField(field, g.cache)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n))
+
+        slice.sons[2] = newIntLit(g, n.info, 0)
+      # the array itself does not need to go through a thread local variable:
+      slice.sons[1] = genDeref(indirectAccess(castExpr, field, n.info))
+
+      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldB.typ,
+                                    indirectAccess(castExpr, fieldB, n.info),
+                                    useShallowCopy=true)
+      slice.sons[3] = threadLocal.newSymNode
+      call.add slice
+    elif (let size = computeSize(g.config, argType); size < 0 or size > 16) and
+        n.getRoot != nil:
+      # it is more efficient to pass a pointer instead:
+      let a = genAddrOf(n)
+      field.typ = a.typ
+      objType.addField(field, g.cache)
+      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
+      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, field.typ,
+                                    indirectAccess(castExpr, field, n.info),
+                                    useShallowCopy=true)
+      call.add(genDeref(threadLocal.newSymNode))
+    else:
+      # boring case
+      field.typ = argType
+      objType.addField(field, g.cache)
+      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n)
+      let threadLocal = addLocalVar(g, varSection, varInit,
+                                    objType.owner, field.typ,
+                                    indirectAccess(castExpr, field, n.info),
+                                    useShallowCopy=true)
+      call.add(threadLocal.newSymNode)
+
+proc wrapProcForSpawn*(g: ModuleGraph; owner: PSym; spawnExpr: PNode; retType: PType;
+                       barrier, dest: PNode = nil): PNode =
+  # if 'barrier' != nil, then it is in a 'parallel' section and we
+  # generate quite different code
+  let n = spawnExpr[^2]
+  let spawnKind = spawnResult(retType, barrier!=nil)
+  case spawnKind
+  of srVoid:
+    internalAssert g.config, dest == nil
+    result = newNodeI(nkStmtList, n.info)
+  of srFlowVar:
+    internalAssert g.config, dest == nil
+    result = newNodeIT(nkStmtListExpr, n.info, retType)
+  of srByVar:
+    if dest == nil: localError(g.config, n.info, "'spawn' must not be discarded")
+    result = newNodeI(nkStmtList, n.info)
+
+  if n.kind notin nkCallKinds:
+    localError(g.config, n.info, "'spawn' takes a call expression")
+    return
+  if optThreadAnalysis in g.config.globalOptions:
+    if {tfThread, tfNoSideEffect} * n[0].typ.flags == {}:
+      localError(g.config, n.info, "'spawn' takes a GC safe call expression")
+  var
+    threadParam = newSym(skParam, getIdent(g.cache, "thread"), owner, n.info, g.config.options)
+    argsParam = newSym(skParam, getIdent(g.cache, "args"), owner, n.info, g.config.options)
+  block:
+    let ptrType = getSysType(g, n.info, tyPointer)
+    threadParam.typ = ptrType
+    argsParam.typ = ptrType
+    argsParam.position = 1
+
+  var objType = createObj(g, owner, n.info)
+  incl(objType.flags, tfFinal)
+  let castExpr = createCastExpr(argsParam, objType)
+
+  var scratchObj = newSym(skVar, getIdent(g.cache, "scratch"), owner, n.info, g.config.options)
+  block:
+    scratchObj.typ = objType
+    incl(scratchObj.flags, sfFromGeneric)
+    var varSectionB = newNodeI(nkVarSection, n.info)
+    varSectionB.addVar(scratchObj.newSymNode)
+    result.add varSectionB
+
+  var call = newNodeIT(nkCall, n.info, n.typ)
+  var fn = n.sons[0]
+  # templates and macros are in fact valid here due to the nature of
+  # the transformation:
+  if fn.kind == nkClosure or (fn.typ != nil and fn.typ.callConv == ccClosure):
+    localError(g.config, n.info, "closure in spawn environment is not allowed")
+  if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro,
+                                               skFunc, skMethod, skConverter}):
+    # for indirect calls we pass the function pointer in the scratchObj
+    var argType = n[0].typ.skipTypes(abstractInst)
+    var field = newSym(skField, getIdent(g.cache, "fn"), owner, n.info, g.config.options)
+    field.typ = argType
+    objType.addField(field, g.cache)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
+    fn = indirectAccess(castExpr, field, n.info)
+  elif fn.kind == nkSym and fn.sym.kind == skIterator:
+    localError(g.config, n.info, "iterator in spawn environment is not allowed")
+  elif fn.typ.callConv == ccClosure:
+    localError(g.config, n.info, "closure in spawn environment is not allowed")
+
+  call.add(fn)
+  var varSection = newNodeI(nkVarSection, n.info)
+  var varInit = newNodeI(nkStmtList, n.info)
+  if barrier.isNil:
+    setupArgsForConcurrency(g, n, objType, scratchObj, castExpr, call,
+                            varSection, varInit, result)
+  else:
+    setupArgsForParallelism(g, n, objType, scratchObj, castExpr, call,
+                            varSection, varInit, result)
+
+  var barrierAsExpr: PNode = nil
+  if barrier != nil:
+    let typ = newType(tyPtr, owner)
+    typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ)
+    var field = newSym(skField, getIdent(g.cache, "barrier"), owner, n.info, g.config.options)
+    field.typ = typ
+    objType.addField(field, g.cache)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier)
+    barrierAsExpr = indirectAccess(castExpr, field, n.info)
+
+  var fvField, fvAsExpr: PNode = nil
+  if spawnKind == srFlowVar:
+    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
+    field.typ = retType
+    objType.addField(field, g.cache)
+    fvField = newDotExpr(scratchObj, field)
+    fvAsExpr = indirectAccess(castExpr, field, n.info)
+    # create flowVar:
+    result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1]))
+    if barrier == nil:
+      result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField.info,
+        fvField)
+
+  elif spawnKind == srByVar:
+    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
+    field.typ = newType(tyPtr, objType.owner)
+    field.typ.rawAddSon(retType)
+    objType.addField(field, g.cache)
+    fvAsExpr = indirectAccess(castExpr, field, n.info)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest))
+
+  let wrapper = createWrapperProc(g, fn, threadParam, argsParam,
+                                  varSection, varInit, call,
+                                  barrierAsExpr, fvAsExpr, spawnKind)
+  result.add callCodegenProc(g, "nimSpawn" & $spawnExpr.len, wrapper.info,
+    wrapper.newSymNode, genAddrOf(scratchObj.newSymNode), nil, spawnExpr)
+
+  if spawnKind == srFlowVar: result.add fvField
+

compiler/suggest.nim

@@ -32,11 +32,11 @@
 
 # included from sigmatch.nim
 
-import algorithm, prefixmatches, lineinfos, pathutils, parseutils, linter
+import algorithm, prefixmatches, lineinfos, parseutils, linter
 from wordrecg import wDeprecated, wError, wAddr, wYield, specialWords
 
 when defined(nimsuggest):
-  import passes, tables # importer
+  import passes, tables, pathutils # importer
 
 const
   sep = '\t'

compiler/syntaxes.nim

@@ -10,7 +10,7 @@
 ## Implements the dispatcher for the different parsers.
 
 import
-  strutils, llstream, ast, astalgo, idents, lexer, options, msgs, parser,
+  strutils, llstream, ast, idents, lexer, options, msgs, parser,
   filters, filter_tmpl, renderer, lineinfos, pathutils
 
 type

compiler/transf.nim

@@ -19,9 +19,9 @@
 # * transforms 'defer' into a 'try finally' statement
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, lookups,
-  idents, renderer, types, passes, semfold, magicsys, cgmeth,
-  sempass2, lowerings, injectdestructors, liftlocals,
+  options, ast, astalgo, trees, msgs,
+  idents, renderer, types, semfold, magicsys, cgmeth,
+  lowerings, injectdestructors, liftlocals,
   modulegraphs, lineinfos
 
 proc transformBody*(g: ModuleGraph, prc: PSym, cache = true;

compiler/trees.nim

@@ -10,7 +10,7 @@
 # tree helper routines
 
 import
-  ast, astalgo, lexer, msgs, strutils, wordrecg, idents
+  ast, wordrecg, idents
 
 proc cyclicTreeAux(n: PNode, visited: var seq[PNode]): bool =
   if n == nil: return

compiler/types.nim

@@ -1579,3 +1579,14 @@ proc isException*(t: PType): bool =
     if t.sons[0] == nil: break
     t = skipTypes(t.sons[0], abstractPtrs)
   return false
+
+proc isSinkTypeForParam*(t: PType): bool =
+  # a parameter like 'seq[owned T]' must not be used only once, but its
+  # elements must, so we detect this case here:
+  result = t.skipTypes({tyGenericInst, tyAlias}).kind in {tySink, tyOwned}
+  when false:
+    if isSinkType(t):
+      if t.skipTypes({tyGenericInst, tyAlias}).kind in {tyArray, tyVarargs, tyOpenArray, tySequence}:
+        result = false
+      else:
+        result = true

compiler/typesrenderer.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import renderer, strutils, ast, msgs, types, astalgo
+import renderer, strutils, ast, types
 
 const defaultParamSeparator* = ","
 

compiler/vm.nim

@@ -13,7 +13,7 @@
 import ast except getstr
 
 import
-  strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes,
+  strutils, msgs, vmdef, vmgen, nimsets, types, passes,
   parser, vmdeps, idents, trees, renderer, options, transf, parseutils,
   vmmarshal, gorgeimpl, lineinfos, tables, btrees, macrocacheimpl,
   modulegraphs, sighashes
@@ -102,7 +102,8 @@ template stackTrace(c: PCtx, tos: PStackFrame, pc: int, msg: string) =
 
 proc bailOut(c: PCtx; tos: PStackFrame) =
   stackTrace(c, tos, c.exceptionInstr, "unhandled exception: " &
-             c.currentExceptionA.sons[3].skipColon.strVal)
+             c.currentExceptionA.sons[3].skipColon.strVal &
+             " [" & c.currentExceptionA.sons[2].skipColon.strVal & "]")
 
 when not defined(nimComputedGoto):
   {.pragma: computedGoto.}
@@ -401,7 +402,7 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType):
     else:
       internalError(c.config, "cannot convert to string " & desttyp.typeToString)
   else:
-    case skipTypes(desttyp, abstractRange).kind
+    case skipTypes(desttyp, abstractVarRange).kind
     of tyInt..tyInt64:
       if dest.kind != rkInt:
         myreset(dest); dest.kind = rkInt
@@ -439,7 +440,7 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType):
       else:
         dest.floatVal = src.floatVal
     of tyObject:
-      if srctyp.skipTypes(abstractRange).kind != tyObject:
+      if srctyp.skipTypes(abstractVarRange).kind != tyObject:
         internalError(c.config, "invalid object-to-object conversion")
       # A object-to-object conversion is essentially a no-op
       moveConst(dest, src)
@@ -1199,8 +1200,15 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           tos = savedFrame
           move(regs, tos.slots)
     of opcRaise:
-      let raised = regs[ra].node
+      let raised =
+        # Empty `raise` statement - reraise current exception
+        if regs[ra].kind == rkNone:
+          c.currentExceptionA
+        else:
+          regs[ra].node
       c.currentExceptionA = raised
+      # Set the `name` field of the exception
+      c.currentExceptionA.sons[2].skipColon.strVal = c.currentExceptionA.typ.sym.name.s
       c.exceptionInstr = pc
 
       var frame = tos
@@ -1384,27 +1392,32 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeBC(rkNode)
       let idx = regs[rb].intVal.int
       var dest = regs[ra].node
-      if dest.kind notin {nkEmpty..nkNilLit} and idx <% dest.len:
-        dest.sons[idx] = regs[rc].node
-      else:
+      if nfSem in dest.flags and allowSemcheckedAstModification notin c.config.legacyFeatures:
+        stackTrace(c, tos, pc, "typechecked nodes may not be modified")
+      elif dest.kind in {nkEmpty..nkNilLit} or idx >=% dest.len:
         stackTrace(c, tos, pc, formatErrorIndexBound(idx, dest.len-1))
+      else:
+        dest.sons[idx] = regs[rc].node
     of opcNAdd:
       decodeBC(rkNode)
       var u = regs[rb].node
-      if u.kind notin {nkEmpty..nkNilLit}:
-        u.add(regs[rc].node)
-      else:
+      if nfSem in u.flags and allowSemcheckedAstModification notin c.config.legacyFeatures:
+        stackTrace(c, tos, pc, "typechecked nodes may not be modified")
+      elif u.kind in {nkEmpty..nkNilLit}:
         stackTrace(c, tos, pc, "cannot add to node kind: " & $u.kind)
+      else:
+        u.add(regs[rc].node)
       regs[ra].node = u
     of opcNAddMultiple:
       decodeBC(rkNode)
       let x = regs[rc].node
       var u = regs[rb].node
-      if u.kind notin {nkEmpty..nkNilLit}:
-        # XXX can be optimized:
-        for i in 0..<x.len: u.add(x.sons[i])
-      else:
+      if nfSem in u.flags and allowSemcheckedAstModification notin c.config.legacyFeatures:
+        stackTrace(c, tos, pc, "typechecked nodes may not be modified")
+      elif u.kind in {nkEmpty..nkNilLit}:
         stackTrace(c, tos, pc, "cannot add to node kind: " & $u.kind)
+      else:
+        for i in 0 ..< x.len: u.add(x.sons[i])
       regs[ra].node = u
     of opcNKind:
       decodeB(rkInt)

compiler/vmdef.nim

@@ -10,8 +10,7 @@
 ## This module contains the type definitions for the new evaluation engine.
 ## An instruction is 1-3 int32s in memory, it is a register based VM.
 
-import ast, passes, msgs, idents, intsets, options, modulegraphs, lineinfos,
-  tables, btrees
+import ast, idents, options, modulegraphs, lineinfos
 
 const
   byteExcess* = 128 # we use excess-K for immediates

compiler/vmdeps.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import ast, types, msgs, os, streams, options, idents, lineinfos
+import ast, types, msgs, os, options, idents, lineinfos
 
 proc opSlurp*(file: string, info: TLineInfo, module: PSym; conf: ConfigRef): string =
   try:

compiler/vmgen.nim

@@ -28,10 +28,8 @@
 # this copy depends on the involved types.
 
 import
-  strutils, ast, astalgo, types, msgs, renderer, vmdef,
-  trees, intsets, magicsys, options, lowerings, lineinfos, transf
-import platform
-from os import splitFile
+  strutils, ast, types, msgs, renderer, vmdef,
+  intsets, magicsys, options, lowerings, lineinfos, transf
 
 const
   debugEchoCode* = defined(nimVMDebug)
@@ -2178,7 +2176,7 @@ proc optimizeJumps(c: PCtx; start: int) =
       var d = i + c.code[i].jmpDiff
       for iters in countdown(maxIterations, 0):
         case c.code[d].opcode
-        of opcJmp, opcJmpBack:
+        of opcJmp:
           d = d + c.code[d].jmpDiff
         of opcTJmp, opcFJmp:
           if c.code[d].regA != reg: break

compiler/vmops.nim

@@ -79,6 +79,9 @@ proc getCurrentExceptionMsgWrapper(a: VmArgs) {.nimcall.} =
   setResult(a, if a.currentException.isNil: ""
                else: a.currentException.sons[3].skipColon.strVal)
 
+proc getCurrentExceptionWrapper(a: VmArgs) {.nimcall.} =
+  setResult(a, a.currentException)
+
 proc staticWalkDirImpl(path: string, relative: bool): PNode =
   result = newNode(nkBracket)
   for k, f in walkDir(path, relative):
@@ -132,6 +135,7 @@ proc registerAdditionalOps*(c: PCtx) =
     wrap1s(readFile, ioop)
     wrap2si(readLines, ioop)
     systemop getCurrentExceptionMsg
+    systemop getCurrentException
     registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} =
       setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1)))
     systemop gorgeEx

compiler/writetracking.nim

@@ -15,8 +15,7 @@
 ##   * Computing an aliasing relation based on the assignments. This relation
 ##     is then used to compute the 'writes' and 'escapes' effects.
 
-import intsets, idents, ast, astalgo, trees, renderer, msgs, types, options,
-  lineinfos
+import intsets, idents, ast, trees, msgs, types, options, lineinfos
 
 const
   debug = false

doc/advopt.txt

@@ -120,6 +120,9 @@ Advanced options:
   --errorMax:N              stop compilation after N errors; 0 means unlimited
   --experimental:$1
                             enable experimental language feature
+  --legacy:$2
+                            enable obsolete/legacy language feature
+                            legacy code.
   --newruntime              use an alternative runtime that uses destructors
                             and that uses a shared heap via -d:useMalloc
   --profiler:on|off         enable profiling; requires `import nimprof`, and

doc/destructors.rst

@@ -52,17 +52,12 @@ written as:
       for i in 0..<a.len:
         a.data[i] = b.data[i]
 
-  proc `=move`*[T](a, b: var myseq[T]) =
-    # do nothing for self-assignments:
-    if a.data == b.data: return
+  proc `=sink`*[T](a: var myseq[T]; b: myseq[T]) =
+    # move assignment
     `=destroy`(a)
     a.len = b.len
     a.cap = b.cap
     a.data = b.data
-    # b's elements have been stolen so ensure that the
-    # destructor for b does nothing:
-    b.data = nil
-    b.len = 0
 
   proc add*[T](x: var myseq[T]; y: sink T) =
     if x.len >= x.cap: resize(x)
@@ -128,38 +123,41 @@ The general pattern in ``=destroy`` looks like:
 
 
 
-`=move` hook
+`=sink` hook
 ------------
 
-A `=move` hook moves an object around, the resources are stolen from the source
-and passed to the destination. It must be ensured that source's destructor does
-not free the resources afterwards.
+A `=sink` hook moves an object around, the resources are stolen from the source
+and passed to the destination. It is ensured that source's destructor does
+not free the resources afterwards by setting the object to its default value
+(the value the object's state started in). Setting an object ``x`` back to its
+default value is written as ``wasMoved(x)``.
 
 The prototype of this hook for a type ``T`` needs to be:
 
 .. code-block:: nim
 
-  proc `=move`(dest, source: var T)
+  proc `=sink`(dest: var T; source: T)
 
 
-The general pattern in ``=move`` looks like:
+The general pattern in ``=sink`` looks like:
 
 .. code-block:: nim
 
-  proc `=move`(dest, source: var T) =
-    # protect against self-assignments:
-    if dest.field != source.field:
-      `=destroy`(dest)
-      dest.field = source.field
-      source.field = nil
+  proc `=sink`(dest: var T; source: T) =
+    `=destroy`(dest)
+    dest.field = source.field
 
 
+**Note**: ``=sink`` does not need to check for self-assignments.
+How self-assignments are handled is explained later in this document.
+
 
 `=` (copy) hook
 ---------------
 
 The ordinary assignment in Nim conceptually copies the values. The ``=`` hook
-is called for assignments that couldn't be transformed into moves.
+is called for assignments that couldn't be transformed into ``=sink``
+operations.
 
 The prototype of this hook for a type ``T`` needs to be:
 
@@ -197,11 +195,11 @@ Swap
 ====
 
 The need to check for self-assignments and also the need to destroy previous
-objects inside ``=`` and ``=move`` is a strong indicator to treat ``system.swap``
-as a builtin primitive of its own that simply swaps every field in the involved
-objects via ``copyMem`` or a comparable mechanism.
+objects inside ``=`` and ``=sink`` is a strong indicator to treat
+``system.swap`` as a builtin primitive of its own that simply swaps every
+field in the involved objects via ``copyMem`` or a comparable mechanism.
 In other words, ``swap(a, b)`` is **not** implemented
-as ``let tmp = move(a); b = move(a); a = move(tmp)``!
+as ``let tmp = move(a); b = move(a); a = move(tmp)``.
 
 This has further consequences:
 
@@ -214,8 +212,12 @@ Sink parameters
 ===============
 
 To move a variable into a collection usually ``sink`` parameters are involved.
-A location that is passed to a ``sink`` parameters should not be used afterwards.
-This is ensured by a static analysis over a control flow graph. A sink parameter
+A location that is passed to a ``sink`` parameter should not be used afterwards.
+This is ensured by a static analysis over a control flow graph. If it cannot be
+proven to be the last usage of the location, a copy is done instead and this
+copy is then passed to the sink parameter.
+
+A sink parameter
 *may* be consumed once in the proc's body but doesn't have to be consumed at all.
 The reason for this is that signatures
 like ``proc put(t: var Table; k: sink Key, v: sink Value)`` should be possible
@@ -250,63 +252,10 @@ An implementation is allowed, but not required to implement even more move
 optimizations (and the current implementation does not).
 
 
-Self assignments
-================
-
-Unfortunately this document departs significantly from
-the older design as specified here, https://github.com/nim-lang/Nim/wiki/Destructors.
-The reason is that under the old design so called "self assignments" could not work.
-
-
-.. code-block:: nim
-
-  proc select(cond: bool; a, b: sink string): string =
-    if cond:
-      result = a # moves a into result
-    else:
-      result = b # moves b into result
-
-  proc main =
-    var x = "abc"
-    var y = "xyz"
-
-    # possible self-assignment:
-    x = select(rand() < 0.5, x, y)
-    # 'select' must communicate what parameter has been
-    # consumed. We cannot simply generate:
-    # (select(...); wasMoved(x); wasMoved(y))
-
-Consequence: ``sink`` parameters for objects that have a non-trivial destructor
-must be passed as by-pointer under the hood. A further advantage is that parameters
-are never destroyed, only variables are. The caller's location passed to
-a ``sink`` parameter has to be destroyed by the caller and does not burden
-the callee.
-
-
-Const temporaries
-=================
-
-Constant literals like ``nil`` cannot be easily be ``=moved``'d. The solution
-is to pass a temporary location that contains ``nil`` to the sink location.
-In other words, ``var T`` can only bind to locations, but ``sink T`` can bind
-to values.
-
-For example:
-
-.. code-block:: nim
-
-  var x: owned ref T = nil
-  # gets turned into by the compiler:
-  var tmp = nil
-  move(x, tmp)
-
 
 Rewrite rules
 =============
 
-**Note**: A function call ``f()`` is always the "last read" of the involved
-temporary location and so covered under the more general rewrite rules.
-
 **Note**: There are two different allowed implementation strategies:
 
 1. The produced ``finally`` section can be a single section that is wrapped
@@ -324,15 +273,28 @@ not destroyed at the scope exit, but at the proc exit.
   finally: `=destroy`(x)
 
 
-  f(...)
-  ------------------------    (function-call)
-  (let tmp = f(...); tmp)
+  g(f(...))
+  ------------------------    (nested-function-call)
+  g(let tmp;
+  bitwiseCopy tmp, f(...);
+  tmp)
   finally: `=destroy`(tmp)
 
 
+  x = f(...)
+  ------------------------    (function-sink)
+  `=sink`(x, f(...))
+
+
   x = lastReadOf z
   ------------------          (move-optimization)
-  `=move`(x, z)
+  `=sink`(x, z)
+  wasMoved(z)
+
+
+  v = v
+  ------------------   (self-assignment-removal)
+  discard "nop"
 
 
   x = y
@@ -340,62 +302,106 @@ not destroyed at the scope exit, but at the proc exit.
   `=`(x, y)
 
 
-  x = move y
-  ------------------          (enforced-move)
-  `=move`(x, y)
+  f_sink(g())
+  -----------------------     (call-to-sink)
+  f_sink(g())
 
 
   f_sink(notLastReadOf y)
-  -----------------------     (copy-to-sink)
-  (let tmp; `=`(tmp, y); f_sink(tmp))
-  finally: `=destroy`(tmp)
+  --------------------------     (copy-to-sink)
+  (let tmp; `=`(tmp, y);
+  f_sink(tmp))
 
 
-  f_sink(move y)
-  -----------------------     (enforced-move-to-sink)
-  (let tmp; `=move`(tmp, y); f_sink(tmp))
-  finally: `=destroy`(tmp)
+  f_sink(lastReadOf y)
+  -----------------------     (move-to-sink)
+  f_sink(y)
+  wasMoved(y)
 
 
+Object and array construction
+=============================
 
-Cursor variables
+Object and array construction is treated as a function call where the
+function has ``sink`` parameters.
+
+
+Destructor removal
+==================
+
+``wasMoved(x);`` followed by a `=destroy(x)` operation cancel each other
+out. An implementation is encouraged to exploit this in order to improve
+efficiency and code sizes.
+
+
+Self assignments
 ================
 
-There is an additional rewrite rule for so called "cursor" variables.
-A cursor variable is a variable that is only used for navigation inside
-a data structure. The otherwise implied copies (or moves) and destructions
-can be avoided altogether for cursor variables:
+``=sink`` in combination with ``wasMoved`` can handle self-assignments but
+it's subtle.
 
-::
+The simple case of ``x = x`` cannot be turned
+into ``=sink(x, x); wasMoved(x)`` because that would lose ``x``'s value.
+The solution is that simple self-assignments are simply transformed into
+an empty statement that does nothing.
 
-  var x {.cursor.}: T
-  x = path(z)
-  stmts
-  --------------------------  (cursor-var)
-  x = bitwiseCopy(path z)
-  stmts
-  # x is not destroyed.
+The complex case looks like a variant of ``x = f(x)``, we consider
+``x = select(rand() < 0.5, x, y)`` here:
 
 
-``stmts`` must not mutate ``z`` nor ``x``. All assignments to ``x`` must be
-of the form ``path(z)`` but the ``z`` can differ. Neither ``z`` nor ``x``
-can be aliased; this implies the addresses of these locations must not be
-used explicitly.
-
-The current implementation does not compute cursor variables but supports
-the ``.cursor`` pragma annotation. Cursor variables are respected and
-simply trusted: No checking is performed that no mutations or aliasing
-occurs.
-
-Cursor variables are commonly used in ``iterator`` implementations:
-
 .. code-block:: nim
 
-  iterator nonEmptyItems(x: seq[string]): string =
-    for i in 0..high(x):
-      let it {.cursor.} = x[i] # no string copies, no destruction of 'it'
-      if it.len > 0:
-        yield it
+  proc select(cond: bool; a, b: sink string): string =
+    if cond:
+      result = a # moves a into result
+    else:
+      result = b # moves b into result
+
+  proc main =
+    var x = "abc"
+    var y = "xyz"
+    # possible self-assignment:
+    x = select(true, x, y)
+
+
+Is transformed into:
+
+
+.. code-block:: nim
+
+  proc select(cond: bool; a, b: sink string): string =
+    try:
+      if cond:
+        `=sink`(result, a)
+        wasMoved(a)
+      else:
+        `=sink`(result, b)
+        wasMoved(b)
+    finally:
+      `=destroy`(b)
+      `=destroy`(a)
+
+  proc main =
+    var
+      x: string
+      y: string
+    try:
+      `=sink`(x, "abc")
+      `=sink`(y, "xyz")
+      `=sink`(x, select(true,
+        let blitTmp = x
+        wasMoved(x)
+        blitTmp,
+        let blitTmp = y
+        wasMoved(y)
+        blitTmp))
+      echo [x]
+    finally:
+      `=destroy`(y)
+      `=destroy`(x)
+
+As can be manually verified, this transformation is correct for
+self-assignments.
 
 
 Lent type
@@ -423,14 +429,14 @@ for expressions of type ``lent T`` or of type ``var T``.
   proc construct(kids: sink seq[Tree]): Tree =
     result = Tree(kids: kids)
     # converted into:
-    `=move`(result.kids, kids)
+    `=sink`(result.kids, kids); wasMoved(kids)
 
   proc `[]`*(x: Tree; i: int): lent Tree =
     result = x.kids[i]
     # borrows from 'x', this is transformed into:
     result = addr x.kids[i]
     # This means 'lent' is like 'var T' a hidden pointer.
-    # Unlike 'var' this cannot be used to mutate the object.
+    # Unlike 'var' this hidden pointer cannot be used to mutate the object.
 
   iterator children*(t: Tree): lent Tree =
     for x in t.kids: yield x
@@ -457,11 +463,10 @@ Let ``W`` be an ``owned ref`` type. Conceptually its hooks look like:
 
   proc `=`(x: var W; y: W) {.error: "owned refs can only be moved".}
 
-  proc `=move`(x, y: var W) =
-    if x != y:
-      `=destroy`(x)
-      bitwiseCopy x, y # raw pointer copy
-      y = nil
+  proc `=sink`(x: var W; y: W) =
+    `=destroy`(x)
+    bitwiseCopy x, y # raw pointer copy
+
 
 Let ``U`` be an unowned ``ref`` type. Conceptually its hooks look like:
 
@@ -477,9 +482,8 @@ Let ``U`` be an unowned ``ref`` type. Conceptually its hooks look like:
     if x != nil: dec x.refcount
     bitwiseCopy x, y # raw pointer copy
 
-  proc `=move`(x, y: var U) =
-    # Note: Moves are the same as assignments.
-    `=`(x, y)
+  proc `=sink`(x: var U, y: U) {.error.}
+  # Note: Moves are not available.
 
 
 Hook lifting
@@ -488,7 +492,7 @@ Hook lifting
 The hooks of a tuple type ``(A, B, ...)`` are generated by lifting the
 hooks of the involved types ``A``, ``B``, ... to the tuple type. In
 other words, a copy ``x = y`` is implemented
-as ``x[0] = y[0]; x[1] = y[1]; ...``, likewise for ``=move`` and ``=destroy``.
+as ``x[0] = y[0]; x[1] = y[1]; ...``, likewise for ``=sink`` and ``=destroy``.
 
 Other value-based compound types like ``object`` and ``array`` are handled
 correspondingly. For ``object`` however, the compiler generated hooks
@@ -527,6 +531,8 @@ have been derived from the rewrite rules and are as follows:
   hooks are generated for ``typeof(x)``.
 - In ``x = ...`` (assignment) hooks are generated for ``typeof(x)``.
 - In ``f(...)`` (function call) hooks are generated for ``typeof(f(...))``.
+- For every sink parameter ``x: sink T`` the hooks are generated
+  for ``typeof(x)``.
 
 
 nodestroy pragma

doc/manual.rst

@@ -95,8 +95,8 @@ and code execution in the executable.
 
 The compiler parses Nim source code into an internal data structure called the
 `abstract syntax tree`:idx: (`AST`:idx:). Then, before executing the code or
-compiling it into the executable, it transforms the AST through `semantic
-analysis`:idx:. This adds semantic information such as expression types,
+compiling it into the executable, it transforms the AST through
+`semantic analysis`:idx:. This adds semantic information such as expression types,
 identifier meanings, and in some cases expression values. An error detected
 during semantic analysis is called a `static error`:idx:. Errors described in
 this manual are static errors when not otherwise specified.
@@ -1518,8 +1518,10 @@ For a ``ref object`` type ``system.new`` is invoked implicitly.
 
 Object variants
 ---------------
-Often an object hierarchy is overkill in certain situations where simple
-variant types are needed.
+Often an object hierarchy is overkill in certain situations where simple variant
+types are needed. Object variants are tagged unions discriminated via a
+enumerated type used for runtime type flexibility, mirroring the concepts of
+*sum types* and *algebraic data types (ADTs)* as found in other languages.
 
 An example:
 
@@ -4406,7 +4408,21 @@ Procedures utilizing type classes in such manner are considered to be
 `implicitly generic`:idx:. They will be instantiated once for each unique
 combination of param types used within the program.
 
-Nim also allows for type classes and regular types to be specified
+Whilst the syntax of type classes appears to resemble that of ADTs/algebraic data
+types in ML-like languages, it should be understood that type classes are static
+constraints to be enforced at type instantations. Type classes are not really
+types in themsleves, but are instead a system of providing generic "checks" that
+ultimately *resolve* to some singular type. Type classes do not allow for
+runtime type dynamism, unlike object variants or methods.
+
+As an example, the following would not compile:
+
+.. code-block:: nim
+  type TypeClass = int | string
+  var foo: TypeClass = 2 # foo's type is resolved to an int here
+  foo = "this will fail" # error here, because foo is an int
+
+Nim allows for type classes and regular types to be specified
 as `type constraints`:idx: of the generic type parameter:
 
 .. code-block:: nim

koch.nim

@@ -10,7 +10,7 @@
 #
 
 const
-  NimbleStableCommit = "d15c8530cb7480ce39ffa85a2dd9819d2d4fc645" # 0.10.2
+  NimbleStableCommit = "da82e3111e662fc1b12f96b3cddd66c749c0f686" # master
 
 when defined(gcc) and defined(windows):
   when defined(x86):

lib/core/allocators.nim

@@ -13,10 +13,10 @@ type
     ZerosMem    ## the allocator always zeros the memory on an allocation
   Allocator* = ptr AllocatorObj
   AllocatorObj* {.inheritable, compilerproc.} = object
-    alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall, raises: [], tags: [].}
-    dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall, raises: [], tags: [].}
-    realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall, raises: [], tags: [].}
-    deallocAll*: proc (a: Allocator) {.nimcall, raises: [], tags: [].}
+    alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall, raises: [], tags: [], gcsafe.}
+    dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall, raises: [], tags: [], gcsafe.}
+    realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall, raises: [], tags: [], gcsafe.}
+    deallocAll*: proc (a: Allocator) {.nimcall, raises: [], tags: [], gcsafe.}
     flags*: set[AllocatorFlag]
     name*: cstring
     allocCount: int

lib/core/macros.nim

@@ -1420,7 +1420,7 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} =
   if n.kind == nnkIntLit: n.intVal != 0
   else: n == bindSym"true" # hacky solution for now
 
-macro expandMacros*(body: typed): typed =
+macro expandMacros*(body: typed): untyped =
   ## Expands one level of macro - useful for debugging.
   ## Can be used to inspect what happens when a macro call is expanded,
   ## without altering its result.

lib/core/runtime_v2.nim

@@ -84,7 +84,7 @@ proc nimRawDispose(p: pointer) {.compilerRtl.} =
       dealloc(p -! sizeof(RefHeader))
     if allocs > 0:
       when hasThreadSupport:
-        atomicDec allocs
+        discard atomicDec(allocs)
       else:
         dec allocs
     else:

lib/core/seqs.nim

@@ -138,7 +138,7 @@ proc shrink*[T](x: var seq[T]; newLen: Natural) =
   mixin `=destroy`
   sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'"
   when not supportsCopyMem(T):
-    for i in countdown(x.len - 1, newLen - 1):
+    for i in countdown(x.len - 1, newLen):
       `=destroy`(x[i])
   # XXX This is wrong for const seqs that were moved into 'x'!
   cast[ptr NimSeqV2[T]](addr x).len = newLen

lib/core/strs.nim

@@ -189,3 +189,13 @@ proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} =
       a.p.cap = b.len
     a.len = b.len
     copyMem(unsafeAddr a.p.data[0], unsafeAddr b.p.data[0], b.len+1)
+
+proc nimPrepareStrMutationV2(s: var NimStringV2) {.compilerRtl.} =
+  if s.p != nil and s.p.allocator == nil:
+    let oldP = s.p
+    # can't mutate a literal, so we need a fresh copy here:
+    let allocator = getLocalAllocator()
+    s.p = cast[ptr NimStrPayload](allocator.alloc(allocator, contentSize(s.len)))
+    s.p.allocator = allocator
+    s.p.cap = s.len
+    copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], s.len+1)

lib/impure/db_postgres.nim

@@ -392,7 +392,8 @@ proc getRow*(db: DbConn, query: SqlQuery,
   var res = setupQuery(db, query, args)
   var L = pqnfields(res)
   result = newRow(L)
-  setRow(res, result, 0, L)
+  if pqntuples(res) > 0:
+    setRow(res, result, 0, L)
   pqclear(res)
 
 proc getRow*(db: DbConn, stmtName: SqlPrepared,
@@ -400,7 +401,8 @@ proc getRow*(db: DbConn, stmtName: SqlPrepared,
   var res = setupQuery(db, stmtName, args)
   var L = pqNfields(res)
   result = newRow(L)
-  setRow(res, result, 0, L)
+  if pqntuples(res) > 0:
+    setRow(res, result, 0, L)
   pqClear(res)
 
 proc getAllRows*(db: DbConn, query: SqlQuery,

lib/impure/db_sqlite.nim

@@ -103,7 +103,7 @@
 
 {.deadCodeElim: on.}  # dce option deprecated
 
-import strutils, sqlite3
+import sqlite3
 
 import db_common
 export db_common
@@ -112,7 +112,7 @@ type
   DbConn* = PSqlite3  ## Encapsulates a database connection.
   Row* = seq[string]  ## A row of a dataset. `NULL` database values will be
                       ## converted to an empty string.
-  InstantRow* = Pstmt ## A handle that can be used to get a row's column
+  InstantRow* = PStmt ## A handle that can be used to get a row's column
                       ## text on demand.
 
 proc dbError*(db: DbConn) {.noreturn.} =
@@ -169,7 +169,7 @@ proc tryExec*(db: DbConn, query: SqlQuery,
   ##    db.close()
   assert(not db.isNil, "Database not connected.")
   var q = dbFormat(query, args)
-  var stmt: sqlite3.Pstmt
+  var stmt: sqlite3.PStmt
   if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK:
     let x = step(stmt)
     if x in {SQLITE_DONE, SQLITE_ROW}:
@@ -198,12 +198,12 @@ proc newRow(L: int): Row =
   for i in 0..L-1: result[i] = ""
 
 proc setupQuery(db: DbConn, query: SqlQuery,
-                args: varargs[string]): Pstmt =
+                args: varargs[string]): PStmt =
   assert(not db.isNil, "Database not connected.")
   var q = dbFormat(query, args)
   if prepare_v2(db, q, q.len.cint, result, nil) != SQLITE_OK: dbError(db)
 
-proc setRow(stmt: Pstmt, r: var Row, cols: cint) =
+proc setRow(stmt: PStmt, r: var Row, cols: cint) =
   for col in 0'i32..cols-1:
     setLen(r[col], column_bytes(stmt, col)) # set capacity
     setLen(r[col], 0)
@@ -506,7 +506,7 @@ proc tryInsertID*(db: DbConn, query: SqlQuery,
   ##    db.close()
   assert(not db.isNil, "Database not connected.")
   var q = dbFormat(query, args)
-  var stmt: sqlite3.Pstmt
+  var stmt: sqlite3.PStmt
   result = -1
   if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK:
     if step(stmt) == SQLITE_DONE:

lib/js/dom.nim

@@ -408,6 +408,7 @@ type
     maxWidth*: cstring
     minHeight*: cstring
     minWidth*: cstring
+    opacity*: cstring
     overflow*: cstring
     padding*: cstring
     paddingBottom*: cstring

lib/nimhcr.nim

@@ -120,10 +120,10 @@
 #       - state in static libs gets duplicated
 #       - linking is slow and therefore iteration time suffers
 #         - have just a single .dll for all .nim files and bulk reload?
-#   - think about the compile/link/passC/passL/emit/injectStmt pragmas
-#     - if a passC pragma is introduced (either written or dragged in by a new
+#   - think about the compile/link/passc/passl/emit/injectStmt pragmas
+#     - if a passc pragma is introduced (either written or dragged in by a new
 #       import) the whole command line for compilation changes - for example:
-#         winlean.nim: {.passC: "-DWIN32_LEAN_AND_MEAN".}
+#         winlean.nim: {.passc: "-DWIN32_LEAN_AND_MEAN".}
 #   - play with plugins/dlls/lfIndirect/lfDynamicLib/lfExportLib - shouldn't add an extra '*'
 #   - everything thread-local related
 # - tests

lib/pure/asyncftpclient.nim

@@ -21,13 +21,13 @@
 ## In order to begin any sort of transfer of files you must first
 ## connect to an FTP server. You can do so with the ``connect`` procedure.
 ##
-##   .. code-block::nim
-##      import asyncdispatch, asyncftpclient
-##      proc main() {.async.} =
-##        var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
-##        await ftp.connect()
-##        echo("Connected")
-##      waitFor(main())
+## .. code-block::nim
+##    import asyncdispatch, asyncftpclient
+##    proc main() {.async.} =
+##      var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
+##      await ftp.connect()
+##      echo("Connected")
+##    waitFor(main())
 ##
 ## A new ``main`` async procedure must be declared to allow the use of the
 ## ``await`` keyword. The connection will complete asynchronously and the
@@ -41,16 +41,16 @@
 ## working directory before you do so with the ``pwd`` procedure, you can also
 ## instead specify an absolute path.
 ##
-##   .. code-block::nim
-##      import asyncdispatch, asyncftpclient
-##      proc main() {.async.} =
-##        var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
-##        await ftp.connect()
-##        let currentDir = await ftp.pwd()
-##        assert currentDir == "/home/user/"
-##        await ftp.store("file.txt", "file.txt")
-##        echo("File finished uploading")
-##      waitFor(main())
+## .. code-block::nim
+##    import asyncdispatch, asyncftpclient
+##    proc main() {.async.} =
+##      var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
+##      await ftp.connect()
+##      let currentDir = await ftp.pwd()
+##      assert currentDir == "/home/user/"
+##      await ftp.store("file.txt", "file.txt")
+##      echo("File finished uploading")
+##    waitFor(main())
 ##
 ## Checking the progress of a file transfer
 ## ========================================
@@ -59,20 +59,20 @@
 ## by specifying a ``onProgressChanged`` procedure to the ``store`` or
 ## ``retrFile`` procedures.
 ##
-##   .. code-block::nim
-##      import asyncdispatch, asyncftpclient
+## .. code-block::nim
+##    import asyncdispatch, asyncftpclient
 ##
-##      proc onProgressChanged(total, progress: BiggestInt,
-##                              speed: float): Future[void] =
-##        echo("Uploaded ", progress, " of ", total, " bytes")
-##        echo("Current speed: ", speed, " kb/s")
+##    proc onProgressChanged(total, progress: BiggestInt,
+##                            speed: float): Future[void] =
+##      echo("Uploaded ", progress, " of ", total, " bytes")
+##      echo("Current speed: ", speed, " kb/s")
 ##
-##      proc main() {.async.} =
-##        var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
-##        await ftp.connect()
-##        await ftp.store("file.txt", "/home/user/file.txt", onProgressChanged)
-##        echo("File finished uploading")
-##      waitFor(main())
+##    proc main() {.async.} =
+##      var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test")
+##      await ftp.connect()
+##      await ftp.store("file.txt", "/home/user/file.txt", onProgressChanged)
+##      echo("File finished uploading")
+##    waitFor(main())
 
 
 import asyncdispatch, asyncnet, nativesockets, strutils, parseutils, os, times