compilation cache: mostly working; generics not yet

compiler/ast.nim

@@ -568,7 +568,7 @@ const
   ConstantDataTypes*: TTypeKinds = {tyArrayConstr, tyArray, tySet, 
                                     tyTuple, tySequence}
   ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, skIterator, 
-    skMacro, skTemplate, skConverter, skStub}
+    skMacro, skTemplate, skConverter, skEnumField, skStub}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfAllConst}
   namePos* = 0
   genericParamsPos* = 1

compiler/rodread.nim

@@ -142,7 +142,7 @@ type
   PRodReader* = ref TRodReader
 
 const 
-  FileVersion* = "1024"       # modify this if the rod-format changes!
+  FileVersion* = "1026"       # modify this if the rod-format changes!
 
 var rodCompilerprocs*: TStrTable
 
@@ -657,17 +657,26 @@ proc decodeSymSafePos(rd: PRodReader, offset: int, info: TLineInfo): PSym =
   result = decodeSym(rd, info)
   rd.pos = oldPos
 
+proc findSomeWhere(id: int) =
+  for i in countup(0, high(gMods)): 
+    var rd = gMods[i].rd
+    if rd != nil: 
+      var d = IITableGet(rd.index.tab, id)
+      if d != invalidKey:
+        echo "found id ", id, " in ", gMods[i].filename
+
 proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym = 
   result = PSym(IdTableGet(r.syms, id))
   if result == nil: 
     # load the symbol:
     var d = IITableGet(r.index.tab, id)
     if d == invalidKey: 
+      # import from other module:
       var moduleID = IiTableGet(r.imports.tab, id)
-      if moduleID < 0: 
+      if moduleID < 0:
         var x = ""
         encodeVInt(id, x)
-        InternalError(info, "missing from both indexes: +" & x) 
+        InternalError(info, "missing from both indexes: +" & x)
       # find the reader with the correct moduleID:
       for i in countup(0, high(gMods)): 
         var rd = gMods[i].rd
@@ -680,6 +689,7 @@ proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
             else:
               var x = ""
               encodeVInt(id, x)
+              when false: findSomeWhere(id)
               InternalError(info, "rrGetSym: no reader found: +" & x)
           else:
             #if IiTableGet(rd.index.tab, id) <> invalidKey then

compiler/rodwrite.nim

@@ -322,7 +322,10 @@ proc addToIndex(w: var TIndex, key, val: int) =
   w.lastIdxVal = val
   IiTablePut(w.tab, key, val)
 
-#var debugWritten = initIntSet()
+const debugWrittenIds = false
+
+when debugWrittenIds:
+  var debugWritten = initIntSet()
 
 proc symStack(w: PRodWriter) =
   var i = 0
@@ -335,10 +338,12 @@ proc symStack(w: PRodWriter) =
         # put definition in here
         var L = w.data.len
         addToIndex(w.index, s.id, L) 
-        #intSetIncl(debugWritten, s.id)
+        when debugWrittenIds: incl(debugWritten, s.id)
         encodeSym(w, s, w.data)
         add(w.data, rodNL)
-        if sfExported in s.flags and s.kind in ExportableSymKinds: 
+        # put into interface section if appropriate:
+        if {sfExported, sfFromGeneric} * s.flags == {sfExported} and 
+            s.kind in ExportableSymKinds: 
           encodeStr(s.name.s, w.interf)
           add(w.interf, ' ')
           encodeVInt(s.id, w.interf)
@@ -355,12 +360,14 @@ proc symStack(w: PRodWriter) =
           if w.methods.len != 0: add(w.methods, ' ')
           encodeVInt(s.id, w.methods)
       elif IiTableGet(w.imports.tab, s.id) == invalidKey: 
-        addToIndex(w.imports, s.id, m.id) #if not Contains(debugWritten, s.id):
-                                          #  MessageOut(w.filename);
-                                          #  debug(s.owner);
-                                          #  debug(s);
-                                          #  InternalError('BUG!!!!');
-                                          #end
+        addToIndex(w.imports, s.id, m.id)
+        when debugWrittenIds:
+          if not Contains(debugWritten, s.id):
+            echo(w.filename)
+            debug(s)
+            debug(s.owner)
+            debug(m)
+            InternalError("BUG!!!!")
     inc(i)
   setlen(w.sstack, 0)
 

compiler/sem.nim

@@ -126,13 +126,13 @@ proc semConstBoolExpr(c: PContext, n: PNode): PNode =
 include semtypes, semexprs, semgnrc, semstmts
 
 proc addCodeForGenerics(c: PContext, n: PNode) = 
-  for i in countup(lastGenericIdx, Len(generics) - 1):
-    var prc = generics[i].instSym
+  for i in countup(c.generics.lastGenericIdx, Len(c.generics.generics) - 1):
+    var prc = c.generics.generics[i].instSym
     if prc.kind in {skProc, skMethod, skConverter} and prc.magic == mNone: 
       if prc.ast == nil or prc.ast.sons[codePos] == nil: 
         InternalError(prc.info, "no code for " & prc.name.s)
       addSon(n, prc.ast)
-  lastGenericIdx = Len(generics)
+  c.generics.lastGenericIdx = Len(c.generics.generics)
 
 proc semExprNoFlags(c: PContext, n: PNode): PNode {.procvar.} = 
   result = semExpr(c, n, {})
@@ -162,7 +162,7 @@ proc myOpenCached(module: PSym, filename: string,
 proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = 
   result = semStmt(c, n)
   # BUGFIX: process newly generated generics here, not at the end!
-  if lastGenericIdx < Len(generics):
+  if c.generics.lastGenericIdx < Len(c.generics.generics):
     var a = newNodeI(nkStmtList, n.info)
     addCodeForGenerics(c, a)
     if sonsLen(a) > 0: 

compiler/semdata.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-# This module contains the data structures for the semantic checking phase.
+## This module contains the data structures for the semantic checking phase.
 
 import 
   strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab,
@@ -37,10 +37,24 @@ type
     genericSym*, instSym*: PSym
     concreteTypes*: seq[PType]
   
+  # If we generate an instance of a generic, we'd like to re-use that
+  # instance if possible across module boundaries. However, this is not
+  # possible if the compilation cache is enabled. So we give up then and use
+  # the caching of generics only per module, not per project.
+  TGenericsCache* {.final.} = object
+    InstTypes*: TIdTable # map PType to PType
+    generics*: seq[TInstantiatedSymbol] # a list of the things to compile
+    lastGenericIdx*: int      # used for the generics stack
+  
+  PGenericsCache* = ref TGenericsCache
   PContext* = ref TContext
   TContext* = object of TPassContext # a context represents a module
     module*: PSym             # the module sym belonging to the context
     p*: PProcCon              # procedure context
+    generics*: PGenericsCache # may point to a global or module-local structure 
+    friendModule*: PSym       # current friend module; may access private data;
+                              # this is used so that generic instantiations can
+                              # access private object fields
     InstCounter*: int         # to prevent endless instantiations
    
     threadEntries*: TSymSeq   # list of thread entries to check
@@ -56,10 +70,13 @@ type
     filename*: string         # the module's filename
     userPragmas*: TStrTable
   
+var
+  gGenericsCache: PGenericsCache # save for modularity
 
-var gInstTypes*: TIdTable # map PType to PType
-var generics*: seq[TInstantiatedSymbol] = @[] # a list of the things to compile
-var lastGenericIdx*: int      # used for the generics stack
+proc newGenericsCache: PGenericsCache =
+  new(result)
+  initIdTable(result.InstTypes)
+  result.generics = @[]
 
 proc newContext*(module: PSym, nimfile: string): PContext
 
@@ -126,11 +143,21 @@ proc newContext(module: PSym, nimfile: string): PContext =
   initLinkedList(result.libs)
   append(result.optionStack, newOptionEntry())
   result.module = module
+  result.friendModule = module
   result.threadEntries = @[]
   result.converters = @[]
   result.filename = nimfile
   result.includedFiles = initIntSet()
   initStrTable(result.userPragmas)
+  if optSymbolFiles notin gGlobalOptions:
+    # re-usage of generic instantiations across module boundaries is
+    # very nice for code size:
+    if gGenericsCache == nil: gGenericsCache = newGenericsCache()
+    result.generics = gGenericsCache
+  else:
+    # we have to give up and use a per-module cache for generic instantiations:
+    result.generics = newGenericsCache()
+    assert gGenericsCache == nil
 
 proc addConverter(c: PContext, conv: PSym) = 
   var L = len(c.converters)
@@ -189,5 +216,4 @@ proc checkSonsLen*(n: PNode, length: int) =
   
 proc checkMinSonsLen*(n: PNode, length: int) = 
   if sonsLen(n) < length: illFormedAst(n)
-  
-initIdTable(gInstTypes)
+

compiler/semexprs.nim

@@ -53,8 +53,9 @@ proc inlineConst(n: PNode, s: PSym): PNode {.inline.} =
 proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = 
   case s.kind
   of skProc, skMethod, skIterator, skConverter: 
+    var smoduleId = getModule(s).id
     if sfProcVar notin s.flags and s.typ.callConv == ccDefault and
-        getModule(s).id != c.module.id: 
+        smoduleId != c.module.id and smoduleId != c.friendModule.id: 
       LocalError(n.info, errXCannotBePassedToProcVar, s.name.s)
     result = symChoice(c, n, s)
   of skConst:
@@ -673,8 +674,10 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
       if ty.sons[0] == nil: break 
       ty = skipTypes(ty.sons[0], {tyGenericInst})
     if f != nil: 
-      if sfExported in f.flags or getModule(f).id == c.module.id: 
-        # is the access to a public field or in the same module?
+      var fmoduleId = getModule(f).id
+      if sfExported in f.flags or fmoduleId == c.module.id or
+          fmoduleId == c.friendModule.id: 
+        # is the access to a public field or in the same module or in a friend?
         n.sons[0] = makeDeref(n.sons[0])
         n.sons[1] = newSymNode(f) # we now have the correct field
         n.typ = f.typ

compiler/seminst.nim

@@ -22,9 +22,9 @@ proc instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable,
     if q.typ.kind notin {tyTypeDesc, tyGenericParam}: continue 
     var s = newSym(skType, q.name, getCurrOwner())
     s.info = q.info
-    incl(s.flags, sfUsed)
+    s.flags = s.flags + {sfUsed, sfFromGeneric}
     var t = PType(IdTableGet(pt, q.typ))
-    if t == nil: 
+    if t == nil:
       LocalError(a.info, errCannotInstantiateX, s.name.s)
       break
     if t.kind == tyGenericParam: 
@@ -45,9 +45,9 @@ proc sameInstantiation(a, b: TInstantiatedSymbol): bool =
     result = true
 
 proc GenericCacheGet(c: PContext, entry: var TInstantiatedSymbol): PSym = 
-  for i in countup(0, Len(generics) - 1):
-    if sameInstantiation(entry, generics[i]):
-      result = generics[i].instSym
+  for i in countup(0, Len(c.generics.generics) - 1):
+    if sameInstantiation(entry, c.generics.generics[i]):
+      result = c.generics.generics[i].instSym
       # checking for the concrete parameter list is wrong and unnecessary!
       #if equalParams(b.typ.n, instSym.typ.n) == paramsEqual:
       #echo "found in cache: ", getProcHeader(result)
@@ -86,9 +86,9 @@ proc instantiateBody(c: PContext, n: PNode, result: PSym) =
     popProcCon(c)
 
 proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
-  for i in countup(0, Len(generics) - 1):
-    if generics[i].genericSym.id == s.id:
-      var oldPrc = generics[i].instSym
+  for i in countup(0, Len(c.generics.generics) - 1):
+    if c.generics.generics[i].genericSym.id == s.id:
+      var oldPrc = c.generics.generics[i].instSym
       pushInfoContext(oldPrc.info)
       openScope(c.tab)
       var n = oldPrc.ast
@@ -112,10 +112,9 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   if c.InstCounter > 1000: InternalError(fn.ast.info, "nesting too deep")
   inc(c.InstCounter)
   # NOTE: for access of private fields within generics from a different module
-  # and other identifiers we fake the current module temporarily!
-  # XXX bad hack!
-  var oldMod = c.module
-  c.module = getModule(fn)
+  # we set the friend module:
+  var oldFriend = c.friendModule
+  c.friendModule = getModule(fn)
   result = copySym(fn, false)
   incl(result.flags, sfFromGeneric)
   result.owner = getCurrOwner().owner
@@ -144,7 +143,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   ParamsTypeCheck(c, result.typ)
   var oldPrc = GenericCacheGet(c, entry)
   if oldPrc == nil:
-    generics.add(entry)
+    c.generics.generics.add(entry)
     if n.sons[pragmasPos].kind != nkEmpty:
       pragma(c, result, n.sons[pragmasPos], allRoutinePragmas)
     instantiateBody(c, n, result)
@@ -154,7 +153,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   popInfoContext()
   closeScope(c.tab)           # close scope for parameters
   popOwner()
-  c.module = oldMod
+  c.friendModule = oldFriend
   dec(c.InstCounter)
   
 proc instGenericContainer(c: PContext, n: PNode, header: PType): PType = 

compiler/semtypes.nim

@@ -527,8 +527,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
       typ = paramType(c, a.sons[length-2], genericParams, cl)
       #if matchType(typ, [(tyVar, 0)], tyGenericInvokation):
       #  debug a.sons[length-2][0][1]
-        
-    else: 
+    else:
       typ = nil
     if a.sons[length-1].kind != nkEmpty:
       def = semExprWithType(c, a.sons[length-1]) 

compiler/semtypinst.nim

@@ -105,7 +105,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
   var header: PType = nil
   when true:
     # search for some instantiation here:
-    result = searchInstTypes(gInstTypes, t)
+    result = searchInstTypes(cl.c.generics.InstTypes, t)
     if result != nil: return
     for i in countup(1, sonsLen(t) - 1):
       var x = t.sons[i]
@@ -116,7 +116,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
         #idTablePut(cl.typeMap, body.sons[i-1], x)
     if header != nil:
       # search again after first pass:
-      result = searchInstTypes(gInstTypes, header)
+      result = searchInstTypes(cl.c.generics.InstTypes, header)
       if result != nil: return
     else:
       header = copyType(t, t.owner, false)
@@ -124,7 +124,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
     # we need to add the candidate here, before it's fully instantiated for
     # recursive instantions:
     result = newType(tyGenericInst, t.sons[0].owner)
-    idTablePut(gInstTypes, header, result)
+    idTablePut(cl.c.generics.InstTypes, header, result)
 
     for i in countup(1, sonsLen(t) - 1):
       var x = replaceTypeVarsT(cl, t.sons[i])
@@ -154,14 +154,14 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
       assert x.kind != tyGenericInvokation
       idTablePut(cl.typeMap, body.sons[i-1], x)
     if header == nil: header = t
-    result = searchInstTypes(gInstTypes, header)
+    result = searchInstTypes(cl.c.generics.InstTypes, header)
     if result != nil: return 
     result = newType(tyGenericInst, t.sons[0].owner)
     for i in countup(0, sonsLen(t) - 1): 
       # if one of the params is not concrete, we cannot do anything
       # but we already raised an error!
       addSon(result, header.sons[i])
-    idTablePut(gInstTypes, header, result)
+    idTablePut(cl.c.generics.InstTypes, header, result)
     var newbody = ReplaceTypeVarsT(cl, lastSon(body))
     newbody.flags = newbody.flags + t.flags + body.flags
     newbody.n = ReplaceTypeVarsN(cl, lastSon(body).n)

compiler/sigmatch.nim

@@ -7,8 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
-# This module implements the signature matching for resolving
-# the call to overloaded procs, generic procs and operators.
+## This module implements the signature matching for resolving
+## the call to overloaded procs, generic procs and operators.
 
 import 
   intsets, ast, astalgo, semdata, types, msgs, renderer, lookups, semtypinst, 

doc/intern.txt

@@ -160,6 +160,9 @@ program*.
 Frontend issues
 ---------------
 
+Methods and type converters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Nimrod contains language features that are *global*. The best example for that
 are multi methods: Introducing a new method with the same name and some 
 compatible object parameter means that the method's dispatcher needs to take
@@ -189,6 +192,17 @@ Both the multi method and the type converter problems are solved by storing
 them in special sections in the ROD file that are loaded *unconditionally*
 when the ROD file is read.
 
+Generics
+~~~~~~~~
+
+If we generate an instance of a generic, we'd like to re-use that
+instance if possible across module boundaries. However, this is not
+possible if the compilation cache is enabled. So we give up then and use
+the caching of generics only per module, not per project. This means that
+``--symbolFiles:on`` hurts a bit for efficiency. A better solution would
+be to persist the instantiations in a global cache per project. This might be
+implemented in later versions.
+
 
 Backend issues
 --------------

lib/pure/parseutils.nim

@@ -274,30 +274,30 @@ type
     ikDollar,                ## escaped ``$`` part of the interpolated string
     ikVar,                   ## ``var`` part of the interpolated string
     ikExpr                   ## ``expr`` part of the interpolated string
-
+
 iterator interpolatedFragments*(s: string): tuple[kind: TInterpolatedKind,
-  value: string] =
-  ## Tokenizes the string `s` into substrings for interpolation purposes.
-  ##
-  ## Example:
-  ##
-  ## .. code-block:: nimrod
-  ##   for k, v in interpolatedFragments("  $this is ${an  example}  $$"):
-  ##     echo "(", k, ", \"", v, "\")"
-  ##
-  ## Results in:
-  ##
-  ## .. code-block:: nimrod
-  ##   (ikString, "  ")
-  ##   (ikExpr, "this")
-  ##   (ikString, " is ")
-  ##   (ikExpr, "an  example")
+  value: string] =
+  ## Tokenizes the string `s` into substrings for interpolation purposes.
+  ##
+  ## Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   for k, v in interpolatedFragments("  $this is ${an  example}  $$"):
+  ##     echo "(", k, ", \"", v, "\")"
+  ##
+  ## Results in:
+  ##
+  ## .. code-block:: nimrod
   ##   (ikString, "  ")
-  ##   (ikDollar, "$")
+  ##   (ikExpr, "this")
+  ##   (ikString, " is ")
+  ##   (ikExpr, "an  example")
+  ##   (ikString, "  ")
+  ##   (ikDollar, "$")
   var i = 0
-  var kind: TInterpolatedKind
-  while true:
-    var j = i
+  var kind: TInterpolatedKind
+  while true:
+    var j = i
     if s[j] == '$':
       if s[j+1] == '{':
         inc j, 2
@@ -333,15 +333,15 @@ iterator interpolatedFragments*(s: string): tuple[kind: TInterpolatedKind,
       while j < s.len and s[j] != '$': inc j
       kind = ikStr
     if j > i:
-      # do not copy the trailing } for ikExpr:
-      yield (kind, substr(s, i, j-1-ord(kind == ikExpr)))
-    else:
-      break
-    i = j
+      # do not copy the trailing } for ikExpr:
+      yield (kind, substr(s, i, j-1-ord(kind == ikExpr)))
+    else:
+      break
+    i = j
 
 when isMainModule:
-  for k, v in interpolatedFragments("$test{}  $this is ${an{  example}}  "):
-    echo "(", k, ", \"", v, "\")"
+  for k, v in interpolatedFragments("$test{}  $this is ${an{  example}}  "):
+    echo "(", k, ", \"", v, "\")"
 
 
 {.pop.}

tests/accept/run/mfriends.nim

@@ -0,0 +1,11 @@
+
+type
+  TMyObj = object 
+    x: int
+    
+proc gen*[T](): T = 
+  var d: TMyObj
+  # access private field here
+  d.x = 3
+  result = d.x
+

tests/accept/run/tfriends.nim

@@ -0,0 +1,11 @@
+discard """
+  output: "3"
+"""
+
+# Tests that a generic instantiation from a different module may access
+# private object fields:
+
+import mfriends
+
+echo gen[int]()
+

todo.txt

@@ -16,11 +16,12 @@ Version 0.8.14
 incremental compilation
 -----------------------
 
+- object types need to be compared by container ID!
 - adapt thread var implementation to care about the new merge operation
 - write test cases: needs test script support
-  - test thread var
-  - test DLL interfacing!
   - stress test with whole compiler
+  - test DLL interfacing!
+  - test thread var
 
 - automate tests:
   - test basic recompilation scheme