adds an option to interleave the generated code with snippets from the original source

Lines from the original source are outputted as comments next to line directives. Hopefully, this will make debugging codegen problems easier. Other changes: The frame setup code now uses a single-line C macro. My motivation was to reduce the noise in the generated output and make it easier to step over the boiler-plate code, but counter-intuitively this also improved the overall compilation speed a little bit so I applied the same treatment to line tracking too (this reduces the size of the generated files and the explanation is that probably the I/O overhead dominates the macro expansion costs).
2026-07-19 23:41:29 +00:00 · 2012-12-02 20:36:29 +02:00
parent e9e22ccb2a
commit d0edb1826b
9 changed files with 149 additions and 85 deletions
--- a/compiler/ccgutils.nim
+++ b/compiler/ccgutils.nim
@@ -154,36 +154,11 @@ proc TableGetType*(tab: TIdTable, key: PType): PObject =
        if sameType(t, key): 
          return tab.data[h].val

-proc toCChar*(c: Char): string = 
-  case c
-  of '\0'..'\x1F', '\x80'..'\xFF': result = '\\' & toOctal(c)
-  of '\'', '\"', '\\': result = '\\' & c
-  else: result = $(c)
-
 proc makeSingleLineCString*(s: string): string =
  result = "\""
  for c in items(s):
    result.add(c.toCChar)
  result.add('\"')
-  
-proc makeCString*(s: string): PRope = 
-  # BUGFIX: We have to split long strings into many ropes. Otherwise
-  # this could trigger an InternalError(). See the ropes module for
-  # further information.
-  const 
-    MaxLineLength = 64
-  result = nil
-  var res = "\""
-  for i in countup(0, len(s) - 1):
-    if (i + 1) mod MaxLineLength == 0:
-      add(res, '\"')
-      add(res, tnl)
-      app(result, toRope(res)) # reset:
-      setlen(res, 1)
-      res[0] = '\"'
-    add(res, toCChar(s[i]))
-  add(res, '\"')
-  app(result, toRope(res))

 proc makeLLVMString*(s: string): PRope = 
  const MaxLineLength = 64
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -271,15 +271,18 @@ proc genCLineDir(r: var PRope, info: TLineInfo) =

 proc genLineDir(p: BProc, t: PNode) = 
  var line = t.info.safeLineNm
+  if optEmbedOrigSrc in gGlobalOptions:
+    app(p.s(cpsStmts), con(~"//", t.info.sourceLine, rnl))
  genCLineDir(p.s(cpsStmts), t.info.toFullPath, line)
  if ({optStackTrace, optEndb} * p.Options == {optStackTrace, optEndb}) and
      (p.prc == nil or sfPure notin p.prc.flags): 
    linefmt(p, cpsStmts, "#endb($1);$n", toRope(line))
  elif ({optLineTrace, optStackTrace} * p.Options ==
      {optLineTrace, optStackTrace}) and
-      (p.prc == nil or sfPure notin p.prc.flags): 
-    lineF(p, cpsStmts, "F.line = $1;F.filename = $2;$n", 
-        [toRope(line), makeCString(toFilename(t.info).extractFilename)])
+      (p.prc == nil or sfPure notin p.prc.flags):
+   
+    linefmt(p, cpsStmts, "nimln($1, $2);$n",
+            line.toRope, t.info.quotedFilename)

 include "ccgtypes.nim"

@@ -696,33 +699,17 @@ proc generateHeaders(m: BModule) =
      appf(m.s[cfsHeaders], "$N#include $1$N", [toRope(it.data)])
    it = PStrEntry(it.Next)

-proc getFrameDecl(p: BProc) = 
-  var slots: PRope
-  if p.frameLen > 0: 
-    discard cgsym(p.module, "TVarSlot")
-    slots = ropeff("  TVarSlot s[$1];$n", ", [$1 x %TVarSlot]", 
-                   [toRope(p.frameLen)])
-  else: 
-    slots = nil
-  lineFF(p, cpsLocals, "volatile struct {TFrame* prev;" &
-      "NCSTRING procname;NI line;NCSTRING filename;" &
-      "NI len;$1} F;$n",
-      "%TF = type {%TFrame*, i8*, %NI, %NI$1}$n" & 
-      "%F = alloca %TF$n", [slots])
-  inc(p.labels)
-  prepend(p.s(cpsInit), indentLine(p, ropeff("F.len = $1;$n", 
-      "%LOC$2 = getelementptr %TF %F, %NI 4$n" &
-      "store %NI $1, %NI* %LOC$2$n", [toRope(p.frameLen), toRope(p.labels)])))
-
 proc retIsNotVoid(s: PSym): bool = 
  result = (s.typ.sons[0] != nil) and not isInvalidReturnType(s.typ.sons[0])

-proc initFrame(p: BProc, procname, filename: PRope): PRope = 
-  result = ropecg(p.module, 
-    "\tF.procname = $1;$n" &
-    "\tF.filename = $2;$n" & 
-    "\tF.line = 0;$n" & 
-    "\t#pushFrame((TFrame*)&F);$n", [procname, filename])
+proc initFrame(p: BProc, procname, filename: PRope): PRope =
+  discard cgsym(p.module, "pushFrame")
+  if p.frameLen > 0:
+    discard cgsym(p.module, "TVarSlot")
+    result = rfmt(nil, "\tnimfrs($1, $2, $3)$N",
+                  procname, filename, p.frameLen.toRope)
+  else:
+    result = rfmt(nil, "\tnimfr($1, $2)$N", procname, filename)

 proc deinitFrame(p: BProc): PRope =
  result = rfmt(p.module, "\t#popFrame();$n")
@@ -774,11 +761,9 @@ proc genProcAux(m: BModule, prc: PSym) =
    generatedProc = rfmt(nil, "$N$1 {$N", header)
    app(generatedProc, initGCFrame(p))
    if optStackTrace in prc.options: 
-      getFrameDecl(p)
      app(generatedProc, p.s(cpsLocals))
      var procname = CStringLit(p, generatedProc, prc.name.s)
-      var filename = CStringLit(p, generatedProc, toFilename(prc.info))
-      app(generatedProc, initFrame(p, procname, filename))
+      app(generatedProc, initFrame(p, procname, prc.info.quotedFilename))
    else: 
      app(generatedProc, p.s(cpsLocals))
    if (optProfiler in prc.options) and (gCmd != cmdCompileToLLVM):
@@ -1014,11 +999,6 @@ proc genInitCode(m: BModule) =
  if m.nimTypes > 0: 
    appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", 
          [m.nimTypesName, toRope(m.nimTypes)])
-  if optStackTrace in m.initProc.options and not m.FrameDeclared:
-    # BUT: the generated init code might depend on a current frame, so
-    # declare it nevertheless:
-    m.FrameDeclared = true
-    getFrameDecl(m.initProc)
  
  app(prc, initGCFrame(m.initProc))
 
@@ -1027,11 +1007,16 @@ proc genInitCode(m: BModule) =
  app(prc, m.preInitProc.s(cpsLocals))
  app(prc, genSectionEnd(cpsLocals))

-  if optStackTrace in m.initProc.options and not m.PreventStackTrace: 
-    var procname = CStringLit(m.initProc, prc, m.module.name.s)
-    var filename = CStringLit(m.initProc, prc, toFilename(m.module.info))
-    app(prc, initFrame(m.initProc, procname, filename))
- 
+  if optStackTrace in m.initProc.options and not m.FrameDeclared:
+    # BUT: the generated init code might depend on a current frame, so
+    # declare it nevertheless:
+    m.FrameDeclared = true
+    if not m.PreventStackTrace:
+      var procname = CStringLit(m.initProc, prc, m.module.name.s)
+      app(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename))
+    else:
+      app(prc, ~"\tvolatile TFrame F; F.len = 0;$N")
+    
  app(prc, genSectionStart(cpsInit))
  app(prc, m.preInitProc.s(cpsInit))
  app(prc, m.initProc.s(cpsInit))
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -257,6 +257,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
  of "debuginfo": 
    expectNoArg(switch, arg, pass, info)
    incl(gGlobalOptions, optCDebug)
+  of "embedsrc":
+    expectNoArg(switch, arg, pass, info)
+    incl(gGlobalOptions, optEmbedOrigSrc)
  of "compileonly", "c": 
    expectNoArg(switch, arg, pass, info)
    incl(gGlobalOptions, optCompileOnly)
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -508,16 +508,31 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
    if (xi <= 255): add(tok.literal, Chr(xi))
    else: lexMessage(L, errInvalidCharacterConstant)
  else: lexMessage(L, errInvalidCharacterConstant)
+
+proc newString(s: cstring, l: int): string =
+  ## XXX, how come there is no support for this?
+  result = newString(l)
+  for i in 0 .. <l:
+    result[i] = s[i]
+
+proc HandleCRLF(L: var TLexer, pos: int): int =
+  template registerLine =
+    let col = L.getColNumber(pos)
+    
+    if col > MaxLineLength:
+      lexMessagePos(L, hintLineTooLong, pos)
+
+    if optEmbedOrigSrc in gGlobalOptions:
+      let lineStart = cast[TAddress](L.buf) + L.lineStart
+      let line = newString(cast[cstring](lineStart), col)
+      addSourceLine(L.fileIdx, line)
  
-proc HandleCRLF(L: var TLexer, pos: int): int = 
  case L.buf[pos]
-  of CR: 
-    if getColNumber(L, pos) > MaxLineLength: 
-      lexMessagePos(L, hintLineTooLong, pos)
+  of CR:
+    registerLine()
    result = lexbase.HandleCR(L, pos)
-  of LF: 
-    if getColNumber(L, pos) > MaxLineLength: 
-      lexMessagePos(L, hintLineTooLong, pos)
+  of LF:
+    registerLine()
    result = lexbase.HandleLF(L, pos)
  else: result = pos
  
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -8,7 +8,7 @@
 #

 import
-  options, strutils, os, tables, sockets
+  options, strutils, os, tables, sockets, ropes, platform

 type 
  TMsgKind* = enum 
@@ -403,6 +403,14 @@ type
  TFileInfo*{.final.} = object 
    fullPath*: string          # This is a canonical full filesystem path
    projPath*: string          # This is relative to the project's root
+    
+    quotedName*: PRope         # cached quoted short name for codegen
+                               # purpoes
+    
+    lines*: seq[PRope]         # the source code of the module
+                               #   used for better error messages and
+                               #   embedding the original source in the
+                               #   generated code

  TLineInfo*{.final.} = object # This is designed to be as small as possible,
                               # because it is used
@@ -424,11 +432,40 @@ var
  fileInfos*: seq[TFileInfo] = @[]
  SystemFileIdx*: int32

+proc toCChar*(c: Char): string = 
+  case c
+  of '\0'..'\x1F', '\x80'..'\xFF': result = '\\' & toOctal(c)
+  of '\'', '\"', '\\': result = '\\' & c
+  else: result = $(c)
+
+proc makeCString*(s: string): PRope =
+  # BUGFIX: We have to split long strings into many ropes. Otherwise
+  # this could trigger an InternalError(). See the ropes module for
+  # further information.
+  const 
+    MaxLineLength = 64
+  result = nil
+  var res = "\""
+  for i in countup(0, len(s) - 1):
+    if (i + 1) mod MaxLineLength == 0:
+      add(res, '\"')
+      add(res, tnl)
+      app(result, toRope(res)) # reset:
+      setlen(res, 1)
+      res[0] = '\"'
+    add(res, toCChar(s[i]))
+  add(res, '\"')
+  app(result, toRope(res))
+
+
 proc newFileInfo(fullPath, projPath: string): TFileInfo =
  result.fullPath = fullPath
  #shallow(result.fullPath)
  result.projPath = projPath
  #shallow(result.projPath)
+  result.quotedName = projPath.extractFilename.makeCString
+  if optEmbedOrigSrc in gGlobalOptions or true:
+    result.lines = @[]

 proc fileInfoIdx*(filename: string): int32 =
  var
@@ -524,11 +561,11 @@ proc getInfoContext*(index: int): TLineInfo =
  if i >=% L: result = UnknownLineInfo()
  else: result = msgContext[i]

-proc ToFilename*(info: TLineInfo): string =
+proc toFilename*(info: TLineInfo): string =
  if info.fileIndex < 0: result = "???"
  else: result = fileInfos[info.fileIndex].projPath

-proc ToFilename*(fileIdx: int32): string =
+proc toFilename*(fileIdx: int32): string =
  if fileIdx < 0: result = "???"
  else: result = fileInfos[fileIdx].projPath

@@ -536,7 +573,7 @@ proc toFullPath*(info: TLineInfo): string =
  if info.fileIndex < 0: result = "???"
  else: result = fileInfos[info.fileIndex].fullPath
  
-proc ToLinenumber*(info: TLineInfo): int {.inline.} = 
+proc toLinenumber*(info: TLineInfo): int {.inline.} = 
  result = info.line

 proc toColumn*(info: TLineInfo): int {.inline.} = 
@@ -714,3 +751,28 @@ template AssertNotNil*(e: expr): expr =

 template InternalAssert*(e: bool): stmt =
  if not e: InternalError($InstantiationInfo())
+
+proc addSourceLine*(fileIdx: int32, line: string) =
+  fileInfos[fileIdx].lines.add line.toRope
+
+proc sourceLine*(i: TLineInfo): PRope =
+  if i.fileIndex < 0: return nil
+  InternalAssert i.fileIndex < fileInfos.len and
+                 i.line <= fileInfos[i.fileIndex].lines.len
+
+  result = fileInfos[i.fileIndex].lines[i.line-1]
+
+proc quotedFilename*(i: TLineInfo): PRope =
+  InternalAssert i.fileIndex >= 0
+  result = fileInfos[i.fileIndex].quotedName
+
+ropes.ErrorHandler = proc (err: TRopesError, msg: string, useWarning: bool) =
+  case err
+  of rInvalidFormatStr:
+    internalError("ropes: invalid format string: " & msg)
+  of rTokenTooLong:
+    internalError("ropes: token too long: " & msg)
+  of rCannotOpenFile:
+    rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile,
+               msg)
+ 
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -30,6 +30,7 @@ type                          # please make sure we have under 32 options
    optImplicitStatic,        # optimization: implicit at compile time
                              # evaluation
    optPatterns               # en/disable pattern matching
+
  TOptions* = set[TOption]
  TGlobalOption* = enum       # **keep binary compatible**
    gloptNone, optForceFullMake, optBoehmGC, optRefcGC, optDeadCodeElim, 
@@ -59,8 +60,9 @@ type                          # please make sure we have under 32 options
    optTaintMode,             # taint mode turned on
    optTlsEmulation,          # thread var emulation turned on
    optGenIndex               # generate index file for documentation;
+    optEmbedOrigSrc           # embed the original source in the generated code
                              # also: generate header file
-
+   
  TGlobalOptions* = set[TGlobalOption]
  TCommands* = enum           # Nimrod's commands
                              # **keep binary compatible**
@@ -236,4 +238,5 @@ proc binaryStrSearch*(x: openarray[string], y: string): int =
 template nimdbg*: expr = c.module.fileIdx == gProjectMainIdx
 template cnimdbg*: expr = p.module.module.fileIdx == gProjectMainIdx
 template pnimdbg*: expr = p.lex.fileIdx == gProjectMainIdx
+template lnimdbg*: expr = L.fileIdx == gProjectMainIdx

--- a/compiler/ropes.nim
+++ b/compiler/ropes.nim
@@ -56,7 +56,7 @@
 #  To cache them they are inserted in a `cache` array.

 import 
-  msgs, strutils, platform, hashes, crc, options
+  strutils, platform, hashes, crc, options

 type
  TFormatStr* = string # later we may change it to CString for better
@@ -72,6 +72,11 @@ type
  
  TRopeSeq* = seq[PRope]

+  TRopesError* = enum
+    rCannotOpenFile
+    rInvalidFormatStr
+    rTokenTooLong
+
 proc con*(a, b: PRope): PRope
 proc con*(a: PRope, b: string): PRope
 proc con*(a: string, b: PRope): PRope
@@ -92,6 +97,9 @@ proc RopeInvariant*(r: PRope): bool
  # exported for debugging
 # implementation

+var ErrorHandler*: proc(err: TRopesError, msg: string, useWarning = false)
+  # avoid dependency on msgs.nim
+  
 proc ropeLen(a: PRope): int = 
  if a == nil: result = 0
  else: result = a.length
@@ -225,8 +233,7 @@ proc WriteRope*(head: PRope, filename: string, useWarning = false) =
    if head != nil: WriteRope(f, head)
    close(f)
  else:
-    rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile,
-               filename)
+    ErrorHandler(rCannotOpenFile, filename, useWarning)

 var
  rnl* = tnl.newRope
@@ -255,8 +262,8 @@ proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope =
          inc(i)
          if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break 
        num = j
-        if j > high(args) + 1: 
-          internalError("ropes: invalid format string $" & $(j))
+        if j > high(args) + 1:
+          ErrorHandler(rInvalidFormatStr, $(j))
        else:
          app(result, args[j - 1])
      of 'n':
@@ -265,7 +272,8 @@ proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope =
      of 'N':
        app(result, rnl)
        inc(i)
-      else: InternalError("ropes: invalid format string $" & frmt[i])
+      else:
+        ErrorHandler(rInvalidFormatStr, $(frmt[i]))
    var start = i
    while i < length:
      if frmt[i] != '$': inc(i)
@@ -290,8 +298,8 @@ const

 proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = 
  if r.data != nil:
-    if r.length > bufSize: 
-      internalError("ropes: token too long")
+    if r.length > bufSize:
+      ErrorHandler(rTokenTooLong, r.data)
      return
    var readBytes = readBuffer(bin, buf, r.length)
    result = readBytes == r.length and
--- a/doc/advopt.txt
+++ b/doc/advopt.txt
@@ -52,6 +52,8 @@ Advanced options:
                            (Nimrod, mangled) identifier pairs
  --project                 document the whole project (doc2)
  --lineDir:on|off          generation of #line directive on|off
+  --embedsrc                embeds the original source code as comments
+                            in the generated output
  --threadanalysis:on|off   turn thread analysis on|off
  --tlsEmulation:on|off     turn thread local storage emulation on|off
  --taintMode:on|off        turn taint mode on|off
--- a/lib/nimbase.h
+++ b/lib/nimbase.h
@@ -436,6 +436,17 @@ struct TFrame {
  NI len;
 };

+#define nimfr(proc, file) \
+  volatile TFrame F; \
+  F.procname = proc; F.filename = file; F.line = 0; F.len = 0; pushFrame(&F);
+
+#define nimfrs(proc, file, slots) \
+  volatile struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; TVarSlot s[slots];} F; \
+  F.procname = proc; F.filename = file; F.line = 0; F.len = slots; pushFrame((TFrame*)&F);
+
+#define nimln(n, file) \
+  F.line = n; F.filename = file;
+
 #define NIM_POSIX_INIT  __attribute__((constructor)) 

 #if defined(_MSCVER) && defined(__i386__)