distinction between re.replace and re.replacef; speed improvements for re module

2026-01-02 19:22:40 +00:00 · 2011-03-06 21:16:05 +01:00
parent e424e13bd9
commit 3005955d20
23 changed files with 206 additions and 171 deletions
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2010 Andreas Rumpf
+#        (c) Copyright 2011 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -32,46 +32,57 @@ type
    reIgnoreCase = 0,    ## do caseless matching
    reMultiLine = 1,     ## ``^`` and ``$`` match newlines within data 
    reDotAll = 2,        ## ``.`` matches anything including NL
-    reExtended = 3       ## ignore whitespace and ``#`` comments
+    reExtended = 3,      ## ignore whitespace and ``#`` comments
+    reStudy = 4          ## study the expression (may be omitted if the
+                         ## expression will be used only once)
    
  TRegExDesc {.pure, final.}  = object 
    h: PPcre
+    e: ptr TExtra
    
  TRegEx* = ref TRegExDesc ## a compiled regular expression
    
  EInvalidRegEx* = object of EInvalidValue
    ## is raised if the pattern is no valid regular expression.

+proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = 
+  var e: ref EInvalidRegEx
+  new(e)
+  e.msg = msg
+  raise e
+  
 proc rawCompile(pattern: string, flags: cint): PPcre =
  var
    msg: CString
    offset: cint
-    com = pcre.Compile(pattern, flags, addr(msg), addr(offset), nil)
-  if com == nil:
-    var e: ref EInvalidRegEx
-    new(e)
-    e.msg = $msg & "\n" & pattern & "\n" & repeatChar(offset) & "^\n"
-    raise e
-  return com
+  result = pcre.Compile(pattern, flags, addr(msg), addr(offset), nil)
+  if result == nil:
+    raiseInvalidRegEx($msg & "\n" & pattern & "\n" & repeatChar(offset) & "^\n")

 proc finalizeRegEx(x: TRegEx) = 
  # XXX This is a hack, but PCRE does not export its "free" function properly.
  # Sigh. The hack relies on PCRE's implementation (see ``pcre_get.c``).
  # Fortunately the implementation is unlikely to change. 
  pcre.free_substring(cast[cstring](x.h))
+  if not isNil(x.e):
+    pcre.free_substring(cast[cstring](x.e))

-proc re*(s: string, flags = {reExtended}): TRegEx =
+proc re*(s: string, flags = {reExtended, reStudy}): TRegEx =
  ## Constructor of regular expressions. Note that Nimrod's
  ## extended raw string literals support this syntax ``re"[abc]"`` as
  ## a short form for ``re(r"[abc]")``.
  new(result, finalizeRegEx)
-  result.h = rawCompile(s, cast[cint](flags))
-  
+  result.h = rawCompile(s, cast[cint](flags - {reStudy}))
+  if reStudy in flags:
+    var msg: cstring
+    result.e = pcre.study(result.h, 0, msg)
+    if not isNil(msg): raiseInvalidRegex($msg)
+
 proc matchOrFind(s: string, pattern: TRegEx, matches: var openarray[string],
                 start, flags: cint): cint =
  var
    rawMatches: array[0..maxSubpatterns * 3 - 1, cint]
-    res = pcre.Exec(pattern.h, nil, s, len(s), start, flags,
+    res = pcre.Exec(pattern.h, pattern.e, s, len(s), start, flags,
      cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3)
  if res < 0'i32: return res
  for i in 1..int(res)-1:
@@ -83,13 +94,13 @@ proc matchOrFind(s: string, pattern: TRegEx, matches: var openarray[string],
  
 proc findBounds*(s: string, pattern: TRegEx, matches: var openarray[string],
                 start = 0): tuple[first, last: int] =
-  ## returns the starting position and end position of ``pattern`` in ``s`` 
+  ## returns the starting position and end position of `pattern` in `s` 
  ## and the captured
-  ## substrings in the array ``matches``. If it does not match, nothing
-  ## is written into ``matches`` and (-1,0) is returned.
+  ## substrings in the array `matches`. If it does not match, nothing
+  ## is written into `matches` and ``(-1,0)`` is returned.
  var
    rawMatches: array[0..maxSubpatterns * 3 - 1, cint]
-    res = pcre.Exec(pattern.h, nil, s, len(s), start, 0'i32,
+    res = pcre.Exec(pattern.h, pattern.e, s, len(s), start, 0'i32,
      cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3)
  if res < 0'i32: return (-1, 0)
  for i in 1..int(res)-1:
@@ -98,10 +109,40 @@ proc findBounds*(s: string, pattern: TRegEx, matches: var openarray[string],
    if a >= 0'i32: matches[i-1] = copy(s, int(a), int(b)-1)
    else: matches[i-1] = ""
  return (rawMatches[0].int, rawMatches[1].int - 1)
+  
+proc findBounds*(s: string, pattern: TRegEx, 
+                 matches: var openarray[tuple[first, last: int]],
+                 start = 0): tuple[first, last: int] =
+  ## returns the starting position and end position of ``pattern`` in ``s`` 
+  ## and the captured substrings in the array `matches`. 
+  ## If it does not match, nothing is written into `matches` and
+  ## ``(-1,0)`` is returned.
+  var
+    rawMatches: array[0..maxSubpatterns * 3 - 1, cint]
+    res = pcre.Exec(pattern.h, pattern.e, s, len(s), start, 0'i32,
+      cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3)
+  if res < 0'i32: return (-1, 0)
+  for i in 1..int(res)-1:
+    var a = rawMatches[i * 2]
+    var b = rawMatches[i * 2 + 1]
+    if a >= 0'i32: matches[i-1] = (int(a), int(b)-1)
+    else: matches[i-1] = (-1,0)
+  return (rawMatches[0].int, rawMatches[1].int - 1)

+proc findBounds*(s: string, pattern: TRegEx, 
+                 start = 0): tuple[first, last: int] =
+  ## returns the starting position of `pattern` in `s`. If it does not
+  ## match, ``(-1,0)`` is returned.
+  var
+    rawMatches: array[0..3 - 1, cint]
+    res = pcre.Exec(pattern.h, nil, s, len(s), start, 0'i32,
+      cast[ptr cint](addr(rawMatches)), 3)
+  if res < 0'i32: return (int(res), 0)
+  return (int(rawMatches[0]), int(rawMatches[1]-1))
+  
 proc matchOrFind(s: string, pattern: TRegEx, start, flags: cint): cint =
  var rawMatches: array [0..maxSubpatterns * 3 - 1, cint]
-  result = pcre.Exec(pattern.h, nil, s, len(s), start, flags,
+  result = pcre.Exec(pattern.h, pattern.e, s, len(s), start, flags,
                    cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3)
  if result >= 0'i32:
    result = rawMatches[1] - rawMatches[0]
@@ -139,7 +180,7 @@ proc find*(s: string, pattern: TRegEx, matches: var openarray[string],
  ## is written into ``matches`` and -1 is returned.
  var
    rawMatches: array[0..maxSubpatterns * 3 - 1, cint]
-    res = pcre.Exec(pattern.h, nil, s, len(s), start, 0'i32,
+    res = pcre.Exec(pattern.h, pattern.e, s, len(s), start, 0'i32,
      cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3)
  if res < 0'i32: return res
  for i in 1..int(res)-1:
@@ -219,31 +260,64 @@ proc endsWith*(s: string, suffix: TRegEx): bool =
  for i in 0 .. s.len-1:
    if matchLen(s, suffix, i) == s.len - i: return true

-proc replace*(s: string, sub: TRegEx, by: string): string =
-  ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by`
-  ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples:
+proc replace*(s: string, sub: TRegEx, by = ""): string =
+  ## Replaces `sub` in `s` by the string `by`. Captures cannot be 
+  ## accessed in `by`. Examples:
  ##
  ## .. code-block:: nimrod
-  ##   "var1=key; var2=key2".replace(re"(\w+)'='(\w+)", "$1<-$2$2")
+  ##   "var1=key; var2=key2".replace(re"(\w+)'='(\w+)")
  ##
  ## Results in:
  ##
  ## .. code-block:: nimrod
  ##
-  ##   "var1<-keykey; val2<-key2key2"
+  ##   "; "
+  result = ""
+  var prev = 0
+  while true:
+    var match = findBounds(s, sub, prev)
+    if match.first < 0: break
+    add(result, copy(s, prev, match.first-1))
+    add(result, by)
+    prev = match.last + 1
+  add(result, copy(s, prev))
+  
+proc replacef*(s: string, sub: TRegEx, by: string): string =
+  ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by`
+  ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples:
+  ##
+  ## .. code-block:: nimrod
+  ## "var1=key; var2=key2".replace(re"(\w+)'='(\w+)", "$1<-$2$2")
+  ##
+  ## Results in:
+  ##
+  ## .. code-block:: nimrod
+  ##
+  ## "var1<-keykey; val2<-key2key2"
  result = ""
-  var i = 0
  var caps: array[0..maxSubpatterns-1, string]
-  while i < s.len:
-    var x = matchLen(s, sub, caps, i)
-    if x <= 0:
-      add(result, s[i])
-      inc(i)
-    else:
-      addf(result, by, caps)
-      inc(i, x)
-  # copy the rest:
-  add(result, copy(s, i))
+  var prev = 0
+  while true:
+    var match = findBounds(s, sub, caps, prev)
+    if match.first < 0: break
+    add(result, copy(s, prev, match.first-1))
+    addf(result, by, caps)
+    prev = match.last + 1
+  add(result, copy(s, prev))
+  when false:
+    result = ""
+    var i = 0
+    var caps: array[0..maxSubpatterns-1, string]
+    while i < s.len:
+      var x = matchLen(s, sub, caps, i)
+      if x <= 0:
+        add(result, s[i])
+        inc(i)
+      else:
+        addf(result, by, caps)
+        inc(i, x)
+    # copy the rest:
+    add(result, copy(s, i))
  
 proc parallelReplace*(s: string, subs: openArray[
                      tuple[pattern: TRegEx, repl: string]]): string = 
@@ -376,8 +450,10 @@ when isMainModule:
    assert false
    
  assert "var1=key; var2=key2".endsWith(re"\w+=\w+")
-  assert("var1=key; var2=key2".replace(re"(\w+)=(\w+)", "$1<-$2$2") ==
+  assert("var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") ==
         "var1<-keykey; var2<-key2key2")
+  assert("var1=key; var2=key2".replace(re"(\w+)=(\w+)", "$1<-$2$2") ==
+         "$1<-$2$2; $1<-$2$2")

  for word in split("00232this02939is39an22example111", re"\d+"):
    writeln(stdout, word)
--- a/lib/oldwrappers/tcl.nim
+++ b/lib/oldwrappers/tcl.nim
@@ -37,7 +37,7 @@
 when defined(WIN32): 
  const dllName = "tcl(85|84|83|82|81|80).dll"
 elif defined(macosx): 
-  const dllName = "libtcl(8.5|8.4|8.3|8.2|8.1).dynlib"
+  const dllName = "libtcl(8.5|8.4|8.3|8.2|8.1).dylib"
 else:
  const dllName = "libtcl(8.5|8.4|8.3|8.2|8.1).so.(1|0)"

--- a/lib/pure/algorithm.nim
+++ b/lib/pure/algorithm.nim
@@ -1,7 +1,7 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2010 Andreas Rumpf
+#        (c) Copyright 2011 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
--- a/lib/pure/pegs.nim
+++ b/lib/pure/pegs.nim
@@ -874,7 +874,7 @@ proc endsWith*(s: string, suffix: TPeg, start = 0): bool {.
  for i in start .. s.len-1:
    if matchLen(s, suffix, i) == s.len - i: return true

-proc replace*(s: string, sub: TPeg, by: string): string {.
+proc replacef*(s: string, sub: TPeg, by: string): string {.
  nosideEffect, rtl, extern: "npegs$1".} =
  ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by`
  ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples:
@@ -898,7 +898,23 @@ proc replace*(s: string, sub: TPeg, by: string): string {.
    else:
      addf(result, by, caps)
      inc(i, x)
-  # copy the rest:
+  add(result, copy(s, i))
+
+proc replace*(s: string, sub: TPeg, by = ""): string {.
+  nosideEffect, rtl, extern: "npegs$1".} =
+  ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed
+  ## in `by`.
+  result = ""
+  var i = 0
+  var caps: array[0..maxSubpatterns-1, string]
+  while i < s.len:
+    var x = matchLen(s, sub, caps, i)
+    if x <= 0:
+      add(result, s[i])
+      inc(i)
+    else:
+      addf(result, by, caps)
+      inc(i, x)
  add(result, copy(s, i))
  
 proc parallelReplace*(s: string, subs: openArray[
@@ -1691,7 +1707,7 @@ when isMainModule:
              """
  assert($g2 == "((A B) / (C D))")
  assert match("cccccdddddd", g2)
-  assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") ==
+  assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") ==
         "var1<-keykey; var2<-key2key2")
  assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}")

@@ -1722,7 +1738,7 @@ when isMainModule:
  assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+")
  assert(not match("456678", peg"(\letter)+"))

-  assert("var1 = key; var2 = key2".replace(
+  assert("var1 = key; var2 = key2".replacef(
    peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") ==
         "var1<-keykey;var2<-key2key2")

--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -687,7 +687,7 @@ proc contains*(s: string, chars: set[char]): bool {.noSideEffect.} =
  ## Same as ``find(s, chars) >= 0``.
  return find(s, chars) >= 0

-proc replace*(s, sub, by: string): string {.noSideEffect,
+proc replace*(s, sub: string, by = ""): string {.noSideEffect,
  rtl, extern: "nsuReplaceStr".} =
  ## Replaces `sub` in `s` by the string `by`.
  var a: TSkipTable
@@ -800,7 +800,7 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
  ##   by ``\xHH`` where ``HH`` is its hexadecimal value.
  ## The procedure has been designed so that its output is usable for many
  ## different common syntaxes. The resulting string is prefixed with
-  ## ``prefix`` and suffixed with ``suffix``. Both may be empty strings.
+  ## `prefix` and suffixed with `suffix`. Both may be empty strings.
  result = prefix
  for c in items(s):
    case c
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -65,11 +65,11 @@ proc `not` *(x: bool): bool {.magic: "Not", noSideEffect.}

 proc `and`*(x, y: bool): bool {.magic: "And", noSideEffect.}
  ## Boolean ``and``; returns true iff ``x == y == true``.
-  ## Evaluation is short-circuited: this means that if ``x`` is false,
+  ## Evaluation is lazy: if ``x`` is false,
  ## ``y`` will not even be evaluated.
 proc `or`*(x, y: bool): bool {.magic: "Or", noSideEffect.}
  ## Boolean ``or``; returns true iff ``not (not x and not y)``.
-  ## Evaluation is short-circuited: this means that if ``x`` is true,
+  ## Evaluation is lazy: if ``x`` is true,
  ## ``y`` will not even be evaluated.
 proc `xor`*(x, y: bool): bool {.magic: "Xor", noSideEffect.}
  ## Boolean `exclusive or`; returns true iff ``x != y``.
@@ -623,7 +623,7 @@ template `not_in` * (x, y: expr): expr = not contains(y, x)
 proc `is` *[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.}
 template `is_not` *(x, y: expr): expr = not (x is y)

-proc cmp*[T, S: typeDesc](x: T, y: S): int {.procvar.} =
+proc cmp*[T](x, y: T): int {.procvar.} =
  ## Generic compare proc. Returns a value < 0 iff x < y, a value > 0 iff x > y
  ## and 0 iff x == y. This is useful for writing generic algorithms without
  ## performance loss. This generic implementation uses the `==` and `<`
@@ -1034,9 +1034,6 @@ iterator countup*[S, T](a: S, b: T, step = 1): T {.inline.} =
  while res <= b:
    yield res
    inc(res, step)
-  # we cannot use ``for x in a..b: `` here, because that is not
-  # known in the System module
-

 proc min*(x, y: int): int {.magic: "MinI", noSideEffect.}
 proc min*(x, y: int8): int8 {.magic: "MinI", noSideEffect.}
--- a/lib/system/inclrtl.nim
+++ b/lib/system/inclrtl.nim
@@ -31,7 +31,7 @@ elif defined(useNimRtl):
  when hostOS == "windows": 
    const nimrtl* = "nimrtl.dll"
  elif hostOS == "macosx":
-    const nimrtl* = "nimrtl.dynlib"
+    const nimrtl* = "nimrtl.dylib"
  else: 
    const nimrtl* = "libnimrtl.so"
  {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl.}
--- a/lib/wrappers/claro.nim
+++ b/lib/wrappers/claro.nim
@@ -27,7 +27,7 @@ when defined(windows):
    clarodll = "claro.dll"
 elif defined(macosx): 
  const 
-    clarodll = "libclaro.dynlib"
+    clarodll = "libclaro.dylib"
 else: 
  const 
    clarodll = "libclaro.so"
--- a/lib/wrappers/pcre.nim
+++ b/lib/wrappers/pcre.nim
@@ -39,7 +39,7 @@ when not defined(pcreDll):
  when hostOS == "windows":
    const pcreDll = "pcre3.dll"
  elif hostOS == "macosx":
-    const pcreDll = "libpcre.dynlib"
+    const pcreDll = "libpcre(.3|).dylib"
  else:
    const pcreDll = "libpcre.so(.3|)"

@@ -275,7 +275,7 @@ proc maketables*(): ptr char{.cdecl, importc: "pcre_maketables",
                                       dynlib: pcredll.}
 proc refcount*(a2: ptr TPcre, a3: cint): cint{.cdecl, importc: "pcre_refcount", 
    dynlib: pcredll.}
-proc study*(a2: ptr TPcre, a3: cint, a4: cstringArray): ptr Textra{.cdecl, 
+proc study*(a2: ptr TPcre, a3: cint, a4: var cstring): ptr Textra{.cdecl, 
    importc: "pcre_study", dynlib: pcredll.}
 proc version*(): cstring{.cdecl, importc: "pcre_version", dynlib: pcredll.}

--- a/lib/wrappers/tcl.nim
+++ b/lib/wrappers/tcl.nim
@@ -39,7 +39,7 @@ when defined(WIN32):
    dllName = "tcl(85|84|83|82|81|80).dll"
 elif defined(macosx): 
  const 
-    dllName = "libtcl(8.5|8.4|8.3|8.2|8.1).dynlib"
+    dllName = "libtcl(8.5|8.4|8.3|8.2|8.1).dylib"
 else: 
  const 
    dllName = "libtcl(8.5|8.4|8.3|8.2|8.1).so.(1|0)"
--- a/lib/wrappers/tre.nim
+++ b/lib/wrappers/tre.nim
@@ -10,7 +10,7 @@ when not defined(treDll):
  when hostOS == "windows":
    const treDll = "tre.dll"
  elif hostOS == "macosx":
-    const treDll = "libtre.dynlib"
+    const treDll = "libtre.dylib"
  else:
    const treDll = "libtre.so(.5|)"

--- a/rod/interact.nim
+++ b/rod/interact.nim
@@ -1,15 +0,0 @@
-#
-#
-#           The Nimrod Compiler
-#        (c) Copyright 2008 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-# This file implements interactive sessions.
-
-import 
-  llstream, strutils, ropes, nstrtabs, msgs
-
-# implementation
--- a/rod/main.nim
+++ b/rod/main.nim
@@ -15,7 +15,7 @@ import
  os, lists, condsyms, rodread, rodwrite, ropes, trees, 
  wordrecg, sem, semdata, idents, passes, docgen, extccomp,
  cgen, ecmasgen,
-  platform, interact, nimconf, importer, passaux, depends, transf, evals, types
+  platform, nimconf, importer, passaux, depends, transf, evals, types

 const
  has_LLVM_Backend = false
--- a/rod/msgs.nim
+++ b/rod/msgs.nim
@@ -412,8 +412,7 @@ proc includeFilename*(f: string): int =
    if filenames[i] == f: 
      return i
  result = len(filenames)
-  setlen(filenames, result + 1)
-  filenames[result] = f
+  filenames.add(f)

 proc newLineInfo*(filename: string, line, col: int): TLineInfo = 
  result.fileIndex = includeFilename(filename)
@@ -421,7 +420,7 @@ proc newLineInfo*(filename: string, line, col: int): TLineInfo =
  result.col = int16(col)

 proc ToFilename*(info: TLineInfo): string = 
-  if info.fileIndex == - 1: result = "???"
+  if info.fileIndex < 0: result = "???"
  else: result = filenames[info.fileIndex]
  
 proc ToLinenumber*(info: TLineInfo): int {.inline.} = 
@@ -456,7 +455,7 @@ proc MsgKindToString*(kind: TMsgKind): string =
  result = msgKindToStr[kind]

 proc getMessageStr(msg: TMsgKind, arg: string): string = 
-  result = `%`(msgKindToString(msg), [arg])
+  result = msgKindToString(msg) % [arg]

 type
  TCheckPointResult* = enum 
@@ -489,19 +488,17 @@ proc handleError(msg: TMsgKind, eh: TErrorHandling) =
    elif eh == doRaise:
      raiseRecoverableError()
  
-proc sameLineInfo(a, b: TLineInfo): bool = 
-  result = (a.line == b.line) and (a.fileIndex == b.fileIndex)
+proc `==`(a, b: TLineInfo): bool = 
+  result = a.line == b.line and a.fileIndex == b.fileIndex

 proc writeContext(lastinfo: TLineInfo) = 
-  var info: TLineInfo
-  info = lastInfo
+  var info = lastInfo
  for i in countup(0, len(msgContext) - 1): 
-    if not sameLineInfo(msgContext[i], lastInfo) and
-        not sameLineInfo(msgContext[i], info): 
-      MsgWriteln(`%`(posErrorFormat, [toFilename(msgContext[i]), 
-                                      coordToStr(msgContext[i].line), 
-                                      coordToStr(msgContext[i].col), 
-                                      getMessageStr(errInstantiationFrom, "")]))
+    if msgContext[i] != lastInfo and msgContext[i] != info: 
+      MsgWriteln(posErrorFormat % [toFilename(msgContext[i]), 
+                                   coordToStr(msgContext[i].line), 
+                                   coordToStr(msgContext[i].col), 
+                                   getMessageStr(errInstantiationFrom, "")])
    info = msgContext[i]

 proc rawMessage*(msg: TMsgKind, args: openarray[string]) = 
@@ -539,7 +536,7 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
    frmt = posErrorFormat
    # we try to filter error messages so that not two error message
    # in the same file and line are produced:
-    ignoreMsg = sameLineInfo(lastError, info)
+    ignoreMsg = lastError == info
    lastError = info
  of warnMin..warnMax: 
    ignoreMsg = optWarns notin gOptions or msg notin gNotes
--- a/rod/passaux.nim
+++ b/rod/passaux.nim
@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #

-# implements some little helper passes
+## implements some little helper passes

 import 
  strutils, ast, astalgo, passes, msgs, options
@@ -20,7 +20,11 @@ proc verboseOpen(s: PSym, filename: string): PPassContext =
 proc verboseProcess(context: PPassContext, n: PNode): PNode = 
  result = n
  if context != nil: InternalError("logpass: context is not nil")
-  if gVerbosity == 3: Message(n.info, hintProcessing, $ast.gid)
+  if gVerbosity == 3: 
+    # system.nim deactivates all hints, for verbosity:3 we want the processing
+    # messages nonetheless, so we activate them again unconditionally:
+    incl(msgs.gNotes, hintProcessing)
+    Message(n.info, hintProcessing, $ast.gid)
  
 proc verbosePass*(): TPass = 
  initPass(result)
@@ -28,7 +32,6 @@ proc verbosePass*(): TPass =
  result.process = verboseProcess

 proc cleanUp(c: PPassContext, n: PNode): PNode = 
-  var s: PSym
  result = n                  
  # we cannot clean up if dead code elimination is activated
  if optDeadCodeElim in gGlobalOptions: return 
@@ -36,9 +39,9 @@ proc cleanUp(c: PPassContext, n: PNode): PNode =
  of nkStmtList: 
    for i in countup(0, sonsLen(n) - 1): discard cleanup(c, n.sons[i])
  of nkProcDef, nkMethodDef: 
-    if (n.sons[namePos].kind == nkSym): 
-      s = n.sons[namePos].sym
-      if not (sfDeadCodeElim in getModule(s).flags) and not astNeeded(s): 
+    if n.sons[namePos].kind == nkSym: 
+      var s = n.sons[namePos].sym
+      if sfDeadCodeElim notin getModule(s).flags and not astNeeded(s): 
        s.ast.sons[codePos] = ast.emptyNode # free the memory
  else: 
    nil
--- a/rod/semexprs.nim
+++ b/rod/semexprs.nim
@@ -398,7 +398,8 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
  if (n.sons[0].kind == nkSym) and (n.sons[0].sym.magic in FakeVarParams): 
    # BUGFIX: check for L-Value still needs to be done for the arguments!
    for i in countup(1, sonsLen(n) - 1): 
-      if i < sonsLen(t) and skipTypes(t.sons[i], abstractInst).kind == tyVar: 
+      if i < sonsLen(t) and t.sons[i] != nil and
+          skipTypes(t.sons[i], abstractInst).kind == tyVar: 
        if isAssignable(n.sons[i]) != arLValue: 
          LocalError(n.sons[i].info, errVarForOutParamNeeded)
    return
--- a/rod/seminst.nim
+++ b/rod/seminst.nim
@@ -47,6 +47,22 @@ proc GenericCacheAdd(c: PContext, genericSym, instSym: PSym) =
  addSon(n, newSymNode(instSym))
  addSon(c.generics, n)

+proc removeDefaultParamValues(n: PNode) = 
+  # we remove default params, because they cannot be instantiated properly
+  # and they are not needed anyway for instantiation (each param is already
+  # provided).
+  when false:
+    for i in countup(1, sonsLen(n)-1): 
+      var a = n.sons[i]
+      if a.kind != nkIdentDefs: IllFormedAst(a)
+      var L = a.len
+      if a.sons[L-1].kind != nkEmpty and a.sons[L-2].kind != nkEmpty:
+        # ``param: typ = defaultVal``. 
+        # We don't need defaultVal for semantic checking and it's wrong for
+        # ``cmp: proc (a, b: T): int = cmp``. Hm, for ``cmp = cmp`` that is
+        # not possible... XXX We don't solve this issue here.
+        a.sons[L-1] = ast.emptyNode
+
 proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, 
                      info: TLineInfo): PSym = 
  # generates an instantiated proc
@@ -76,6 +92,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
  n.sons[genericParamsPos] = ast.emptyNode
  # semantic checking for the parameters:
  if n.sons[paramsPos].kind != nkEmpty: 
+    removeDefaultParamValues(n.sons[ParamsPos])
    semParamList(c, n.sons[ParamsPos], nil, result)
    addParams(c, result.typ.n)
  else: 
--- a/rod/semtypes.nim
+++ b/rod/semtypes.nim
@@ -11,7 +11,9 @@

 proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = 
  result = IndexTypesMatch(c, formal, arg.typ, arg)
-  if result == nil: typeMismatch(arg, formal, arg.typ)
+  if result == nil: 
+    #debug(arg)
+    typeMismatch(arg, formal, arg.typ)
  
 proc newOrPrevType(kind: TTypeKind, prev: PType, c: PContext): PType = 
  if prev == nil: 
@@ -518,7 +520,8 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
      # check type compability between def.typ and typ:
      if typ == nil: 
        typ = def.typ
-      elif def != nil and def.typ != nil and def.typ.kind != tyNone:
+      elif def != nil:
+        # and def.typ != nil and def.typ.kind != tyNone:
        # example code that triggers it:
        # proc sort[T](cmp: proc(a, b: T): int = cmp)
        def = fitNode(c, typ, def)
--- a/rod/webrepl.nim
+++ b/rod/webrepl.nim
@@ -1,71 +0,0 @@
-#
-#
-#           The Nimrod Compiler
-#        (c) Copyright 2010 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Creates a server, opens a browser and starts serving a Repl for the user.
-## Unfortunately it doesn't ever stop...
-
-import httpserver, sockets, browsers, strutils, cgi, options
-
-const
-  gui = """
-<html>  
-  <head>
-    <title>Nimrod Interactive Web Console</title>
-  </head>
-  
-  <body>
-    <form action="exec" method="get">
-      <input type="submit" value="Run" /><br />
-      <textarea name="code" cols="80" rows="30">import strutils, os
-
-# your code here</textarea>
-      <table border="0">
-        <tr>
-          <td><input type="checkbox" name="objChecks" checked="true"
-               value="on">objChecks</input></td>
-          <td><input type="checkbox" name="fieldChecks" checked="true"
-               value="on">fieldChecks</input></td>
-          <td><input type="checkbox" name="rangeChecks" checked="true"
-               value="on">rangeChecks</input></td>
-        </tr><tr>
-          <td><input type="checkbox" name="boundChecks" checked="true"
-               value="on">boundChecks</input></td>
-          <td><input type="checkbox" name="overflowChecks" checked="true"
-               value="on">overflowChecks</input></td>
-          <td><input type="checkbox" name="nanChecks" checked="true"
-               value="on">nanChecks</input></td>
-        </tr><tr>
-          <td><input type="checkbox" name="infChecks" checked="true"
-               value="on">infChecks</input></td>
-          <td><input type="checkbox" name="assertions" checked="true"
-               value="on">assertions</input></td>
-        </tr>
-      </table>
-    </form>
-    $1
-  </body>
-</html>
-"""
-
-proc runCode(input: string): string =
-  nil
-
-proc handleRequest(client: TSocket, path, query: string) =
-  var output = query
-  client.send(gui % output & wwwNL)
-
-
-var s: TServer
-open(s, TPort(0))
-browsers.openDefaultBrowser("http://localhost:" & $s.port)
-while true: 
-  next(s)
-  handleRequest(s.client, s.path, s.query)
-  close(s.client)
-close(s)
--- a/tests/accept/compile/tsortdev.nim
+++ b/tests/accept/compile/tsortdev.nim
@@ -50,7 +50,7 @@ when isMainModule:
      for j in 0 .. L-1: 
        data[j] = (math.random(90) - 10)
      var copy = data
-      sort(data, cmp[int, int], order)
+      sort(data, cmp[int], order)
      if not sorted(data, order):
        #for x in items(data): echo x
        break
--- a/todo.txt
+++ b/todo.txt
@@ -8,8 +8,6 @@
 - deprecate ^ and make it available as operator
 - test branch coverage
 - checked exceptions
- do not ambiguity error for methods if ambiguity only affects the same 
-  dispatcher anyway
 - slicing


@@ -109,6 +107,13 @@ Version 2
  
  type
    PWindow = ref TWindow not nil
+    
+  The problem with ``nil`` is that the language currently relies on it for
+  implicit initialization. Initialization is different from assignment. The
+  issues can "easily" dealt with by ensuring:
+  
+    var x = myProc() # checks myProc() initializes every pointer explicitely
+    
 - the two other parsers


--- a/tools/nimgrep.cfg
+++ b/tools/nimgrep.cfg
@@ -0,0 +1,5 @@
+# The GC is still too fragile; I don't want that ``--replace`` does bad things
+# to important files. Nimgrep does not really need a GC anyway.
+
+--gc:none
+
--- a/web/news.txt
+++ b/web/news.txt
@@ -34,7 +34,8 @@ Changes affecting backwards compatibility
  ``module.re"abc"`` is now supported.
 - Changed the behaviour of ``strutils.%``, ``ropes.%`` 
  if both ``$#`` and ``$i`` are involved.
-
+- The ``pegs`` and ``re`` modules distinguish between ``replace`` 
+  and ``replacef`` operations.

 Additions
 ---------