From 76885c754a8f51a0ea34f76dd0843b1949ac7fde Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Dec 2012 00:44:29 +0100 Subject: [PATCH 1/2] first version of ropes.nim with unsafeNew (broken) --- compiler/ccgexprs.nim | 15 +++++-- compiler/ccgmerge.nim | 37 +++++++++++++--- compiler/docgen.nim | 8 ++-- compiler/ropes.nim | 101 ++++++++++++++++++++++-------------------- lib/system.nim | 10 ++++- todo.txt | 2 +- 6 files changed, 110 insertions(+), 63 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 57d06a988a..cb0dcfe00a 100755 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -933,22 +933,31 @@ proc genNew(p: BProc, e: PNode) = var a, b: TLoc reftype, bt: PType + sizeExpr: PRope refType = skipTypes(e.sons[1].typ, abstractVarRange) InitLocExpr(p, e.sons[1], a) initLoc(b, locExpr, a.t, OnHeap) + # 'genNew' also handles 'unsafeNew': + if e.len == 3: + var se: TLoc + InitLocExpr(p, e.sons[2], se) + sizeExpr = se.rdLoc + else: + sizeExpr = ropef("sizeof($1)", + getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange))) let args = [getTypeDesc(p.module, reftype), genTypeInfo(p.module, refType), - getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange))] + sizeExpr] if a.s == OnHeap and optRefcGc in gGlobalOptions: # use newObjRC1 as an optimization; and we don't need 'keepAlive' either if canFormAcycle(a.t): lineCg(p, cpsStmts, "if ($1) #nimGCunref($1);$n", a.rdLoc) else: lineCg(p, cpsStmts, "if ($1) #nimGCunrefNoCycle($1);$n", a.rdLoc) - b.r = ropecg(p.module, "($1) #newObjRC1($2, sizeof($3))", args) + b.r = ropecg(p.module, "($1) #newObjRC1($2, $3)", args) lineCg(p, cpsStmts, "$1 = $2;$n", a.rdLoc, b.rdLoc) else: - b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", args) + b.r = ropecg(p.module, "($1) #newObj($2, $3)", args) genAssignment(p, a, b, {needToKeepAlive}) # set the object type: bt = skipTypes(refType.sons[0], abstractRange) genObjectInit(p, cpsStmts, bt, a, false) diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index df7e7f68b3..027cd5780c 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -145,20 +145,47 @@ proc atEndMark(buf: cstring, pos: int): bool = while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s result = s == NimMergeEndMark.len +when false: + proc readVerbatimSection(L: var TBaseLexer): PRope = + var pos = L.bufpos + var buf = L.buf + result = newMutableRope(30_000) + while true: + case buf[pos] + of CR: + pos = lexbase.HandleCR(L, pos) + buf = L.buf + result.data.add(tnl) + of LF: + pos = lexbase.HandleLF(L, pos) + buf = L.buf + result.data.add(tnl) + of '\0': + InternalError("ccgmerge: expected: " & NimMergeEndMark) + break + else: + if atEndMark(buf, pos): + inc pos, NimMergeEndMark.len + break + result.data.add(buf[pos]) + inc pos + L.bufpos = pos + freezeMutableRope(result) + proc readVerbatimSection(L: var TBaseLexer): PRope = var pos = L.bufpos var buf = L.buf - result = newMutableRope(30_000) + var r = newStringOfCap(30_000) while true: case buf[pos] of CR: pos = lexbase.HandleCR(L, pos) buf = L.buf - result.data.add(tnl) + r.add(tnl) of LF: pos = lexbase.HandleLF(L, pos) buf = L.buf - result.data.add(tnl) + r.add(tnl) of '\0': InternalError("ccgmerge: expected: " & NimMergeEndMark) break @@ -166,10 +193,10 @@ proc readVerbatimSection(L: var TBaseLexer): PRope = if atEndMark(buf, pos): inc pos, NimMergeEndMark.len break - result.data.add(buf[pos]) + r.add(buf[pos]) inc pos L.bufpos = pos - freezeMutableRope(result) + result = r.toRope proc readKey(L: var TBaseLexer, result: var string) = var pos = L.bufpos diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 5d46be8743..e06277f914 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -350,9 +350,11 @@ proc CommandRstAux(filename, outExt: string) = var d = newDocumentor(filen, options.gConfigVars) var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc, {roSupportRawDirective}) - d.modDesc = newMutableRope(30_000) - renderRstToOut(d[], rst, d.modDesc.data) - freezeMutableRope(d.modDesc) + var modDesc = newStringOfCap(30_000) + #d.modDesc = newMutableRope(30_000) + renderRstToOut(d[], rst, modDesc) + #freezeMutableRope(d.modDesc) + d.modDesc = toRope(modDesc) writeOutput(d, filename, outExt) generateIndex(d) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index 50c89e4d92..af05c68a76 100755 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -58,6 +58,9 @@ import msgs, strutils, platform, hashes, crc, options +const + PayloadSize = 64_000 # dummy size for range checking + type TFormatStr* = string # later we may change it to CString for better # performance of the code generator (assignments @@ -66,9 +69,9 @@ type PRope* = ref TRope TRope*{.acyclic.} = object of TObject # the empty rope is represented # by nil to safe space - left*, right*: PRope - length*: int - data*: string # != nil if a leaf + left, right: PRope + L: int # < 0 if a leaf + d: array [0..PayloadSize, char] # != nil if a leaf TRopeSeq* = seq[PRope] @@ -94,22 +97,31 @@ proc RopeInvariant*(r: PRope): bool proc ropeLen(a: PRope): int = if a == nil: result = 0 - else: result = a.length + else: result = a.L.abs -proc newRope(data: string = nil): PRope = - new(result) - if data != nil: - result.length = len(data) - result.data = data +proc newRope(data: string = nil): PRope = + if data != nil: + unsafeNew(result, sizeof(TRope)-PayloadSize+len(data)) + result.L = -len(data) + # copy including '\0': + copyMem(addr result.d, cstring(data), len(data)) + else: + unsafeNew(result, sizeof(TRope)-PayloadSize) -proc newMutableRope*(capacity = 30): PRope = - ## creates a new rope that supports direct modifications of the rope's - ## 'data' and 'length' fields. - new(result) - result.data = newStringOfCap(capacity) +proc eqContent(r: PRope, s: string): bool = + assert r.L < 0 + if -r.L == s.len: + result = equalMem(addr(r.d), cstring(s), s.len) -proc freezeMutableRope*(r: PRope) {.inline.} = - r.length = r.data.len +when false: + proc newMutableRope*(capacity = 30): PRope = + ## creates a new rope that supports direct modifications of the rope's + ## 'data' and 'length' fields. + new(result) + result.data = newStringOfCap(capacity) + + proc freezeMutableRope*(r: PRope) {.inline.} = + r.length = r.data.len var cache: array[0..2048*2 -1, PRope] @@ -130,7 +142,7 @@ proc RopeInvariant(r: PRope): bool = proc insertInCache(s: string): PRope = var h = hash(s) and high(cache) result = cache[h] - if isNil(result) or result.data != s: + if isNil(result) or not eqContent(result, s): result = newRope(s) cache[h] = result @@ -155,19 +167,19 @@ proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = var stack = @[r] while len(stack) > 0: var it = pop(stack) - while it.data == nil: + while it.L >= 0: add(stack, it.right) it = it.left - assert(it.data != nil) - CopyMem(addr(result[resultLen]), addr(it.data[0]), it.length) - Inc(resultLen, it.length) + assert(it.L < 0) + CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L) + Inc(resultLen, -it.L) assert(resultLen <= len(result)) proc ropeToStr(p: PRope): string = if p == nil: result = "" - else: - result = newString(p.length) + else: + result = newString(p.L.abs) var resultLen = 0 newRecRopeToStr(result, resultLen, p) @@ -176,7 +188,7 @@ proc con(a, b: PRope): PRope = elif b == nil: result = a else: result = newRope() - result.length = a.length + b.length + result.L = a.L.abs + b.L.abs result.left = a result.right = b @@ -192,16 +204,16 @@ proc app(a: var PRope, b: PRope) = a = con(a, b) proc app(a: var PRope, b: string) = a = con(a, b) proc prepend(a: var PRope, b: PRope) = a = con(b, a) -proc writeRope*(f: TFile, c: PRope) = +proc writeRope*(f: TFile, c: PRope) = var stack = @[c] - while len(stack) > 0: + while len(stack) > 0: var it = pop(stack) - while it.data == nil: + while it.L >= 0: add(stack, it.right) it = it.left assert(it != nil) - assert(it.data != nil) - write(f, it.data) + assert(it.L < 0) + write(f, cstring(it.d), -it.L) proc WriteRope*(head: PRope, filename: string, useWarning = false) = var f: tfile @@ -261,14 +273,14 @@ const bufSize = 1024 # 1 KB is reasonable proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = - if r.data != nil: - if r.length > bufSize: + if r.L < 0: + if -r.L > bufSize: internalError("ropes: token too long") return - var readBytes = readBuffer(bin, buf, r.length) - result = readBytes == r.length and - equalMem(buf, addr(r.data[0]), r.length) # BUGFIX - else: + var readBytes = readBuffer(bin, buf, -r.L) + result = readBytes == -r.L and + equalMem(buf, addr(r.d[0]), readBytes) + else: result = auxRopeEqualsFile(r.left, bin, buf) if result: result = auxRopeEqualsFile(r.right, bin, buf) @@ -284,29 +296,20 @@ proc RopeEqualsFile(r: PRope, f: string): bool = dealloc(buf) close(bin) -proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = - if r.data != nil: - result = startVal - for i in countup(0, len(r.data) - 1): - result = updateCrc32(r.data[i], result) - else: - result = crcFromRopeAux(r.left, startVal) - result = crcFromRopeAux(r.right, result) - proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = # XXX profiling shows this is actually expensive var stack: TRopeSeq = @[r] result = startVal while len(stack) > 0: var it = pop(stack) - while it.data == nil: + while it.L >= 0: add(stack, it.right) it = it.left - assert(it.data != nil) + assert(it.L < 0) var i = 0 - var L = len(it.data) - while i < L: - result = updateCrc32(it.data[i], result) + var L = -it.L + while i < L: + result = updateCrc32(it.d[i], result) inc(i) proc crcFromRope(r: PRope): TCrc32 = diff --git a/lib/system.nim b/lib/system.nim index 9ad99fb793..e199d7611f 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -112,11 +112,17 @@ proc new*[T](a: var ref T) {.magic: "New", noSideEffect.} ## creates a new object of type ``T`` and returns a safe (traced) ## reference to it in ``a``. -proc new(T: typedesc): ref T = +proc new*(T: typedesc): ref T = ## creates a new object of type ``T`` and returns a safe (traced) ## reference to it as result value new(result) - + +proc unsafeNew*[T](a: var ref T, size: int) {.magic: "New", noSideEffect.} + ## creates a new object of type ``T`` and returns a safe (traced) + ## reference to it in ``a``. This is **unsafe** as it allocates an object + ## of the passed ``size``. This should only be used for optimization + ## purposes when you know what you're doing! + proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.} ## leaked implementation detail. Do not use. diff --git a/todo.txt b/todo.txt index 61b178a230..5c2bea08f4 100755 --- a/todo.txt +++ b/todo.txt @@ -50,7 +50,7 @@ Concurrency provide a ``syncgc`` pragma to trigger compiler injection --> more general: an ``injectLoop`` pragma - 'writes: []' effect; track reads/writes for shared types -- use the effect system for static deadlock prevention +- use the effect system for static deadlock prevention and race detection version 0.9.XX From d7adc7c3289feae7de617953452eb71fb75f40da Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Dec 2012 11:11:54 +0100 Subject: [PATCH 2/2] dont use unsafeNew in ropes.nim for now --- compiler/ropes.nim | 725 +++++++++++++++++++++++++++++++-------------- todo.txt | 2 - web/news.txt | 1 + 3 files changed, 497 insertions(+), 231 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index af05c68a76..f27bf70f72 100755 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -66,54 +66,49 @@ type # performance of the code generator (assignments # copy the format strings # though it is not necessary) - PRope* = ref TRope - TRope*{.acyclic.} = object of TObject # the empty rope is represented - # by nil to safe space - left, right: PRope - L: int # < 0 if a leaf - d: array [0..PayloadSize, char] # != nil if a leaf - - TRopeSeq* = seq[PRope] -proc con*(a, b: PRope): PRope -proc con*(a: PRope, b: string): PRope -proc con*(a: string, b: PRope): PRope -proc con*(a: varargs[PRope]): PRope -proc app*(a: var PRope, b: PRope) -proc app*(a: var PRope, b: string) -proc prepend*(a: var PRope, b: PRope) -proc toRope*(s: string): PRope -proc toRope*(i: BiggestInt): PRope -proc ropeLen*(a: PRope): int -proc writeRopeIfNotEqual*(r: PRope, filename: string): bool -proc ropeToStr*(p: PRope): string -proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope -proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) -proc RopeEqualsFile*(r: PRope, f: string): bool - # returns true if the rope r is the same as the contents of file f -proc RopeInvariant*(r: PRope): bool - # exported for debugging -# implementation +when true: + # working version: + type + PRope* = ref TRope + TRope*{.acyclic.} = object of TObject # the empty rope is represented + # by nil to safe space + left*, right*: PRope + length*: int + data*: string # != nil if a leaf + + TRopeSeq* = seq[PRope] -proc ropeLen(a: PRope): int = - if a == nil: result = 0 - else: result = a.L.abs - -proc newRope(data: string = nil): PRope = - if data != nil: - unsafeNew(result, sizeof(TRope)-PayloadSize+len(data)) - result.L = -len(data) - # copy including '\0': - copyMem(addr result.d, cstring(data), len(data)) - else: - unsafeNew(result, sizeof(TRope)-PayloadSize) + proc con*(a, b: PRope): PRope + proc con*(a: PRope, b: string): PRope + proc con*(a: string, b: PRope): PRope + proc con*(a: varargs[PRope]): PRope + proc app*(a: var PRope, b: PRope) + proc app*(a: var PRope, b: string) + proc prepend*(a: var PRope, b: PRope) + proc toRope*(s: string): PRope + proc toRope*(i: BiggestInt): PRope + proc ropeLen*(a: PRope): int + proc writeRopeIfNotEqual*(r: PRope, filename: string): bool + proc ropeToStr*(p: PRope): string + proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope + proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) + proc RopeEqualsFile*(r: PRope, f: string): bool + # returns true if the rope r is the same as the contents of file f + proc RopeInvariant*(r: PRope): bool + # exported for debugging + # implementation -proc eqContent(r: PRope, s: string): bool = - assert r.L < 0 - if -r.L == s.len: - result = equalMem(addr(r.d), cstring(s), s.len) + proc ropeLen(a: PRope): int = + if a == nil: result = 0 + else: result = a.length + + proc newRope(data: string = nil): PRope = + new(result) + if data != nil: + result.length = len(data) + result.data = data -when false: proc newMutableRope*(capacity = 30): PRope = ## creates a new rope that supports direct modifications of the rope's ## 'data' and 'length' fields. @@ -123,204 +118,476 @@ when false: proc freezeMutableRope*(r: PRope) {.inline.} = r.length = r.data.len -var - cache: array[0..2048*2 -1, PRope] + var + cache: array[0..2048*2 -1, PRope] -proc RopeInvariant(r: PRope): bool = - if r == nil: - result = true - else: - result = true # - # if r.data <> snil then - # result := true - # else begin - # result := (r.left <> nil) and (r.right <> nil); - # if result then result := ropeInvariant(r.left); - # if result then result := ropeInvariant(r.right); - # end + proc RopeInvariant(r: PRope): bool = + if r == nil: + result = true + else: + result = true # + # if r.data <> snil then + # result := true + # else begin + # result := (r.left <> nil) and (r.right <> nil); + # if result then result := ropeInvariant(r.left); + # if result then result := ropeInvariant(r.right); + # end -proc insertInCache(s: string): PRope = - var h = hash(s) and high(cache) - result = cache[h] - if isNil(result) or not eqContent(result, s): - result = newRope(s) - cache[h] = result - -proc toRope(s: string): PRope = - if s.len == 0: - result = nil - else: - result = insertInCache(s) - assert(RopeInvariant(result)) + proc insertInCache(s: string): PRope = + var h = hash(s) and high(cache) + result = cache[h] + if isNil(result) or result.data != s: + result = newRope(s) + cache[h] = result + + proc toRope(s: string): PRope = + if s.len == 0: + result = nil + else: + result = insertInCache(s) + assert(RopeInvariant(result)) -proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = - var length = len(rs) - if at > length: - setlen(rs, at + 1) - else: - setlen(rs, length + 1) # move old rope elements: - for i in countdown(length, at + 1): - rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it - rs[at] = r + proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = + var length = len(rs) + if at > length: + setlen(rs, at + 1) + else: + setlen(rs, length + 1) # move old rope elements: + for i in countdown(length, at + 1): + rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it + rs[at] = r -proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = - var stack = @[r] - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it.L < 0) - CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L) - Inc(resultLen, -it.L) - assert(resultLen <= len(result)) + proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = + var stack = @[r] + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it.data != nil) + CopyMem(addr(result[resultLen]), addr(it.data[0]), it.length) + Inc(resultLen, it.length) + assert(resultLen <= len(result)) -proc ropeToStr(p: PRope): string = - if p == nil: - result = "" - else: - result = newString(p.L.abs) - var resultLen = 0 - newRecRopeToStr(result, resultLen, p) + proc ropeToStr(p: PRope): string = + if p == nil: + result = "" + else: + result = newString(p.length) + var resultLen = 0 + newRecRopeToStr(result, resultLen, p) -proc con(a, b: PRope): PRope = - if a == nil: result = b - elif b == nil: result = a - else: - result = newRope() - result.L = a.L.abs + b.L.abs - result.left = a - result.right = b + proc con(a, b: PRope): PRope = + if a == nil: result = b + elif b == nil: result = a + else: + result = newRope() + result.length = a.length + b.length + result.left = a + result.right = b -proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) -proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) + proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) + proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) -proc con(a: varargs[PRope]): PRope = - for i in countup(0, high(a)): result = con(result, a[i]) + proc con(a: varargs[PRope]): PRope = + for i in countup(0, high(a)): result = con(result, a[i]) -proc toRope(i: BiggestInt): PRope = result = toRope($i) + proc toRope(i: BiggestInt): PRope = result = toRope($i) -proc app(a: var PRope, b: PRope) = a = con(a, b) -proc app(a: var PRope, b: string) = a = con(a, b) -proc prepend(a: var PRope, b: PRope) = a = con(b, a) + proc app(a: var PRope, b: PRope) = a = con(a, b) + proc app(a: var PRope, b: string) = a = con(a, b) + proc prepend(a: var PRope, b: PRope) = a = con(b, a) -proc writeRope*(f: TFile, c: PRope) = - var stack = @[c] - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it != nil) - assert(it.L < 0) - write(f, cstring(it.d), -it.L) + proc writeRope*(f: TFile, c: PRope) = + var stack = @[c] + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it != nil) + assert(it.data != nil) + write(f, it.data) -proc WriteRope*(head: PRope, filename: string, useWarning = false) = - var f: tfile - if open(f, filename, fmWrite): - if head != nil: WriteRope(f, head) - close(f) - else: - rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, - filename) + proc WriteRope*(head: PRope, filename: string, useWarning = false) = + var f: tfile + if open(f, filename, fmWrite): + if head != nil: WriteRope(f, head) + close(f) + else: + rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, + filename) -proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = - var i = 0 - var length = len(frmt) - result = nil - var num = 0 - while i <= length - 1: - if frmt[i] == '$': - inc(i) # skip '$' - case frmt[i] - of '$': - app(result, "$") - inc(i) - of '#': - inc(i) - app(result, args[num]) - inc(num) - of '0'..'9': - var j = 0 - while true: - j = (j * 10) + Ord(frmt[i]) - ord('0') - inc(i) - if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break - num = j - if j > high(args) + 1: - internalError("ropes: invalid format string $" & $(j)) - else: - app(result, args[j - 1]) - of 'n': - if optLineDir notin gOptions: app(result, tnl) - inc i - of 'N': - app(result, tnl) - inc(i) - else: InternalError("ropes: invalid format string $" & frmt[i]) - var start = i - while i < length: - if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: - app(result, substr(frmt, start, i - 1)) - assert(RopeInvariant(result)) - -proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = - app(c, ropef(frmt, args)) - -const - bufSize = 1024 # 1 KB is reasonable - -proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = - if r.L < 0: - if -r.L > bufSize: - internalError("ropes: token too long") - return - var readBytes = readBuffer(bin, buf, -r.L) - result = readBytes == -r.L and - equalMem(buf, addr(r.d[0]), readBytes) - else: - result = auxRopeEqualsFile(r.left, bin, buf) - if result: result = auxRopeEqualsFile(r.right, bin, buf) - -proc RopeEqualsFile(r: PRope, f: string): bool = - var bin: tfile - result = open(bin, f) - if not result: - return # not equal if file does not exist - var buf = alloc(BufSize) - result = auxRopeEqualsFile(r, bin, buf) - if result: - result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? - dealloc(buf) - close(bin) - -proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = - # XXX profiling shows this is actually expensive - var stack: TRopeSeq = @[r] - result = startVal - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it.L < 0) + proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = var i = 0 - var L = -it.L - while i < L: - result = updateCrc32(it.d[i], result) - inc(i) + var length = len(frmt) + result = nil + var num = 0 + while i <= length - 1: + if frmt[i] == '$': + inc(i) # skip '$' + case frmt[i] + of '$': + app(result, "$") + inc(i) + of '#': + inc(i) + app(result, args[num]) + inc(num) + of '0'..'9': + var j = 0 + while true: + j = (j * 10) + Ord(frmt[i]) - ord('0') + inc(i) + if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break + num = j + if j > high(args) + 1: + internalError("ropes: invalid format string $" & $(j)) + else: + app(result, args[j - 1]) + of 'n': + if optLineDir notin gOptions: app(result, tnl) + inc i + of 'N': + app(result, tnl) + inc(i) + else: InternalError("ropes: invalid format string $" & frmt[i]) + var start = i + while i < length: + if frmt[i] != '$': inc(i) + else: break + if i - 1 >= start: + app(result, substr(frmt, start, i - 1)) + assert(RopeInvariant(result)) -proc crcFromRope(r: PRope): TCrc32 = - result = newCrcFromRopeAux(r, initCrc32) + proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = + app(c, ropef(frmt, args)) -proc writeRopeIfNotEqual(r: PRope, filename: string): bool = - # returns true if overwritten - var c: TCrc32 - c = crcFromFile(filename) - if c != crcFromRope(r): - writeRope(r, filename) - result = true - else: - result = false + const + bufSize = 1024 # 1 KB is reasonable + + proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = + if r.data != nil: + if r.length > bufSize: + internalError("ropes: token too long") + return + var readBytes = readBuffer(bin, buf, r.length) + result = readBytes == r.length and + equalMem(buf, addr(r.data[0]), r.length) # BUGFIX + else: + result = auxRopeEqualsFile(r.left, bin, buf) + if result: result = auxRopeEqualsFile(r.right, bin, buf) + + proc RopeEqualsFile(r: PRope, f: string): bool = + var bin: tfile + result = open(bin, f) + if not result: + return # not equal if file does not exist + var buf = alloc(BufSize) + result = auxRopeEqualsFile(r, bin, buf) + if result: + result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? + dealloc(buf) + close(bin) + + proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + if r.data != nil: + result = startVal + for i in countup(0, len(r.data) - 1): + result = updateCrc32(r.data[i], result) + else: + result = crcFromRopeAux(r.left, startVal) + result = crcFromRopeAux(r.right, result) + + proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + # XXX profiling shows this is actually expensive + var stack: TRopeSeq = @[r] + result = startVal + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it.data != nil) + var i = 0 + var L = len(it.data) + while i < L: + result = updateCrc32(it.data[i], result) + inc(i) + + proc crcFromRope(r: PRope): TCrc32 = + result = newCrcFromRopeAux(r, initCrc32) + + proc writeRopeIfNotEqual(r: PRope, filename: string): bool = + # returns true if overwritten + var c: TCrc32 + c = crcFromFile(filename) + if c != crcFromRope(r): + writeRope(r, filename) + result = true + else: + result = false + +else: + # optimized but broken version: + + type + PRope* = ref TRope + TRope*{.acyclic.} = object of TObject # the empty rope is represented + # by nil to safe space + left, right: PRope + L: int # < 0 if a leaf + d: array [0..PayloadSize, char] # != nil if a leaf + + TRopeSeq* = seq[PRope] + + proc con*(a, b: PRope): PRope + proc con*(a: PRope, b: string): PRope + proc con*(a: string, b: PRope): PRope + proc con*(a: varargs[PRope]): PRope + proc app*(a: var PRope, b: PRope) + proc app*(a: var PRope, b: string) + proc prepend*(a: var PRope, b: PRope) + proc toRope*(s: string): PRope + proc toRope*(i: BiggestInt): PRope + proc ropeLen*(a: PRope): int + proc writeRopeIfNotEqual*(r: PRope, filename: string): bool + proc ropeToStr*(p: PRope): string + proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope + proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) + proc RopeEqualsFile*(r: PRope, f: string): bool + # returns true if the rope r is the same as the contents of file f + proc RopeInvariant*(r: PRope): bool + # exported for debugging + # implementation + + proc ropeLen(a: PRope): int = + if a == nil: result = 0 + else: result = a.L.abs + + proc newRope(data: string = nil): PRope = + if data != nil: + unsafeNew(result, sizeof(TRope)-PayloadSize+len(data)+1) + result.L = -len(data) + # copy including '\0': + copyMem(addr result.d, cstring(data), len(data)) + else: + unsafeNew(result, sizeof(TRope)-PayloadSize+1) + + proc eqContent(r: PRope, s: string): bool = + assert r.L < 0 + if -r.L == s.len: + result = equalMem(addr(r.d), cstring(s), s.len) + + when false: + proc newMutableRope*(capacity = 30): PRope = + ## creates a new rope that supports direct modifications of the rope's + ## 'data' and 'length' fields. + new(result) + result.data = newStringOfCap(capacity) + + proc freezeMutableRope*(r: PRope) {.inline.} = + r.length = r.data.len + + var + cache: array[0..2048*2 -1, PRope] + + proc RopeInvariant(r: PRope): bool = + if r == nil: + result = true + else: + result = true # + # if r.data <> snil then + # result := true + # else begin + # result := (r.left <> nil) and (r.right <> nil); + # if result then result := ropeInvariant(r.left); + # if result then result := ropeInvariant(r.right); + # end + + proc insertInCache(s: string): PRope = + var h = hash(s) and high(cache) + result = cache[h] + if isNil(result) or not eqContent(result, s): + result = newRope(s) + cache[h] = result + + proc toRope(s: string): PRope = + if s.len == 0: + result = nil + else: + result = insertInCache(s) + assert(RopeInvariant(result)) + + proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = + var length = len(rs) + if at > length: + setlen(rs, at + 1) + else: + setlen(rs, length + 1) # move old rope elements: + for i in countdown(length, at + 1): + rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it + rs[at] = r + + proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = + var stack = @[r] + while len(stack) > 0: + var it = pop(stack) + while it.L >= 0: + add(stack, it.right) + it = it.left + assert(it.L < 0) + CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L) + Inc(resultLen, -it.L) + assert(resultLen <= len(result)) + + proc ropeToStr(p: PRope): string = + if p == nil: + result = "" + else: + result = newString(p.L.abs) + var resultLen = 0 + newRecRopeToStr(result, resultLen, p) + + proc con(a, b: PRope): PRope = + if a == nil: result = b + elif b == nil: result = a + else: + result = newRope() + result.L = a.L.abs + b.L.abs + result.left = a + result.right = b + + proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) + proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) + + proc con(a: varargs[PRope]): PRope = + for i in countup(0, high(a)): result = con(result, a[i]) + + proc toRope(i: BiggestInt): PRope = result = toRope($i) + + proc app(a: var PRope, b: PRope) = a = con(a, b) + proc app(a: var PRope, b: string) = a = con(a, b) + proc prepend(a: var PRope, b: PRope) = a = con(b, a) + + proc writeRope*(f: TFile, c: PRope) = + var stack = @[c] + while len(stack) > 0: + var it = pop(stack) + while it.L >= 0: + add(stack, it.right) + it = it.left + assert(it != nil) + assert(it.L < 0) + write(f, cstring(it.d), -it.L) + + proc WriteRope*(head: PRope, filename: string, useWarning = false) = + var f: tfile + if open(f, filename, fmWrite): + if head != nil: WriteRope(f, head) + close(f) + else: + rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, + filename) + + proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = + var i = 0 + var length = len(frmt) + result = nil + var num = 0 + while i <= length - 1: + if frmt[i] == '$': + inc(i) # skip '$' + case frmt[i] + of '$': + app(result, "$") + inc(i) + of '#': + inc(i) + app(result, args[num]) + inc(num) + of '0'..'9': + var j = 0 + while true: + j = (j * 10) + Ord(frmt[i]) - ord('0') + inc(i) + if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break + num = j + if j > high(args) + 1: + internalError("ropes: invalid format string $" & $(j)) + else: + app(result, args[j - 1]) + of 'n': + if optLineDir notin gOptions: app(result, tnl) + inc i + of 'N': + app(result, tnl) + inc(i) + else: InternalError("ropes: invalid format string $" & frmt[i]) + var start = i + while i < length: + if frmt[i] != '$': inc(i) + else: break + if i - 1 >= start: + app(result, substr(frmt, start, i - 1)) + assert(RopeInvariant(result)) + + proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = + app(c, ropef(frmt, args)) + + const + bufSize = 1024 # 1 KB is reasonable + + proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = + if r.L < 0: + if -r.L > bufSize: + internalError("ropes: token too long") + return + var readBytes = readBuffer(bin, buf, -r.L) + result = readBytes == -r.L and + equalMem(buf, addr(r.d[0]), readBytes) + else: + result = auxRopeEqualsFile(r.left, bin, buf) + if result: result = auxRopeEqualsFile(r.right, bin, buf) + + proc RopeEqualsFile(r: PRope, f: string): bool = + var bin: tfile + result = open(bin, f) + if not result: + return # not equal if file does not exist + var buf = alloc(BufSize) + result = auxRopeEqualsFile(r, bin, buf) + if result: + result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? + dealloc(buf) + close(bin) + + proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + # XXX profiling shows this is actually expensive + var stack: TRopeSeq = @[r] + result = startVal + while len(stack) > 0: + var it = pop(stack) + while it.L >= 0: + add(stack, it.right) + it = it.left + assert(it.L < 0) + var i = 0 + var L = -it.L + while i < L: + result = updateCrc32(it.d[i], result) + inc(i) + + proc crcFromRope(r: PRope): TCrc32 = + result = newCrcFromRopeAux(r, initCrc32) + + proc writeRopeIfNotEqual(r: PRope, filename: string): bool = + # returns true if overwritten + var c: TCrc32 + c = crcFromFile(filename) + if c != crcFromRope(r): + writeRope(r, filename) + result = true + else: + result = false diff --git a/todo.txt b/todo.txt index 5c2bea08f4..ca2def6aaa 100755 --- a/todo.txt +++ b/todo.txt @@ -15,8 +15,6 @@ version 0.9.2 - ``hoist`` pragma for loop hoisting: can be easily done with AST overloading + global -- implement ``system.unsafeNew``. - version 0.9.X ============= diff --git a/web/news.txt b/web/news.txt index fc54bbce21..1650bd562b 100755 --- a/web/news.txt +++ b/web/news.txt @@ -17,6 +17,7 @@ Library Additions - Added ``system.onRaise`` to support a condition system. - Added ``macros.quote`` for AST quasi-quoting. +- Added ``system.unsafeNew`` to support hacky variable length objects. Changes affecting backwards compatibility