From e20e035f1206fbae88afbbb6673b38ac509a22e9 Mon Sep 17 00:00:00 2001 From: araq Date: Tue, 10 Mar 2026 23:48:46 +0100 Subject: [PATCH] API additions, let's see what it breaks --- compiler/ccgexprs.nim | 19 ++++------- compiler/cgen.nim | 2 +- compiler/llstream.nim | 6 ++-- lib/pure/lexbase.nim | 4 ++- lib/pure/osproc.nim | 4 +-- lib/pure/streams.nim | 25 +++++++------- lib/pure/strutils.nim | 11 ++++--- lib/std/formatfloat.nim | 7 +++- lib/std/private/digitsutils.nim | 2 +- lib/std/strbasics.nim | 6 ++-- lib/std/syncio.nim | 17 +++++----- lib/system.nim | 18 ++++++++-- lib/system/indices.nim | 3 +- lib/system/strs_v2.nim | 17 ++++++++++ lib/system/strs_v3.nim | 58 ++++++++++++++++++++++++++------- 15 files changed, 138 insertions(+), 61 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 96c376ec98..7a3acf4e07 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -988,16 +988,10 @@ proc genAddr(p: BProc, e: PNode, d: var TLoc) = expr(p, e[0], d) # bug #19497 d.lode = e - elif p.config.isDefined("nimsso") and e[0].kind == nkBracketExpr and - e[0][0].typ.skipTypes(abstractVar).kind == tyString: - # addr s[i] for nimsso: nimStrAtMutV3 returns char* directly — no extra & needed - var base = initLocExpr(p, e[0][0]) - var idx = initLocExpr(p, e[0][1]) - putIntoDest(p, d, e, - cCall(cgsymValue(p.module, "nimStrAtMutV3"), byRefLoc(p, base), rdLoc(idx)), - base.storage) else: - var a: TLoc = initLocExpr(p, e[0]) + let ssoStrSub = p.config.isDefined("nimsso") and e[0].kind == nkBracketExpr and + e[0][0].typ.skipTypes(abstractVar).kind == tyString + var a: TLoc = initLocExpr(p, e[0], if ssoStrSub: {lfEnforceDeref} else: {}) if e[0].kind in {nkHiddenStdConv, nkHiddenSubConv, nkConv} and not ignoreConv(e[0]): # addr (conv x) introduces a temp because `conv x` is not a rvalue # transform addr ( conv ( x ) ) -> conv ( addr ( x ) ) @@ -1325,10 +1319,11 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = a.snippet = cDeref(a.snippet) if p.config.isDefined("nimsso") and ty.kind == tyString: - # direct writes (s[i] = c) are intercepted in genAsgn via nimStrPutV3 let bra = byRefLoc(p, a) - if lfPrepareForMutation in d.flags: - # s[i] passed as `var char`: return *(nimStrAtMutV3(&s, i)) — a valid C lvalue + if {lfPrepareForMutation, lfEnforceDeref} * d.flags != {}: + # Use nimStrAtMutV3 to get a mutable reference (char*) to the element. + # Note: for long strings with i < AlwaysAvail the inline cache may become + # stale; callers should use s[i]=c or nimStrPutV3 when possible. putIntoDest(p, d, n, cDeref(cCall(cgsymValue(p.module, "nimStrAtMutV3"), bra, rcb)), a.storage) else: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 756a0ac14d..537d248103 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -389,7 +389,7 @@ proc lenField(p: BProc, val: Rope): Rope {.inline.} = proc lenExpr(p: BProc; a: TLoc): Rope = if optSeqDestructors in p.config.globalOptions: - if p.config.isDefined("nimsso") and a.t != nil and + if p.config.isDefined("nimsso") and a.lode != nil and a.t != nil and a.t.skipTypes(abstractInst).kind == tyString: result = cCall(cgsymValue(p.module, "nimStrLen"), rdLoc(a)) else: diff --git a/compiler/llstream.nim b/compiler/llstream.nim index 9392bb41b2..77cb94eb7e 100644 --- a/compiler/llstream.nim +++ b/compiler/llstream.nim @@ -163,7 +163,8 @@ proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int = inc(s.lineOffset) result = min(bufLen, s.s.len - s.rd) if result > 0: - copyMem(buf, addr(s.s[s.rd]), result) + let (sdata, _) = readRawData(s.s) + copyMem(buf, addr sdata[s.rd], result) inc(s.rd, result) proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int = @@ -173,7 +174,8 @@ proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int = of llsString: result = min(bufLen, s.s.len - s.rd) if result > 0: - copyMem(buf, addr(s.s[0 + s.rd]), result) + let (sdata, _) = readRawData(s.s) + copyMem(buf, addr sdata[s.rd], result) inc(s.rd, result) of llsFile: result = readBuffer(s.f, buf, bufLen) diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 1efd97b244..e36192a6e3 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -65,7 +65,9 @@ proc fillBuffer(L: var BaseLexer) = L.buf[i] = L.buf[L.sentinel + 1 + i] else: # "moveMem" handles overlapping regions - moveMem(addr L.buf[0], addr L.buf[L.sentinel + 1], toCopy) + let p = beginStore(L.buf, L.buf.len) + moveMem(p, addr p[L.sentinel + 1], toCopy) + endStore(L.buf) charsRead = L.input.readDataStr(L.buf, toCopy ..< toCopy + L.sentinel + 1) s = toCopy + charsRead if charsRead < L.sentinel + 1: diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 5718efb51c..502358a85d 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -921,7 +921,7 @@ elif not defined(useNimRtl): for key, val in pairs(t): var x = key & "=" & val result[i] = cast[cstring](alloc(x.len+1)) - copyMem(result[i], addr(x[0]), x.len+1) + copyMem(result[i], x.cstring, x.len+1) inc(i) proc envToCStringArray(): cstringArray = @@ -932,7 +932,7 @@ elif not defined(useNimRtl): for key, val in envPairs(): var x = key & "=" & val result[i] = cast[cstring](alloc(x.len+1)) - copyMem(result[i], addr(x[0]), x.len+1) + copyMem(result[i], x.cstring, x.len+1) inc(i) type diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index a49b2eb0d8..1c31ca0752 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -259,10 +259,8 @@ proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int = result = s.readDataStrImpl(s, buffer, slice) else: # fallback - when declared(prepareMutation): - # buffer might potentially be a CoW literal with ARC - prepareMutation(buffer) - result = s.readData(addr buffer[slice.a], slice.b + 1 - slice.a) + result = s.readData(beginStore(buffer, slice.b + 1 - slice.a, slice.a), slice.b + 1 - slice.a) + endStore(buffer) template jsOrVmBlock(caseJsOrVm, caseElse: untyped): untyped = when nimvm: @@ -1228,8 +1226,9 @@ else: # after 1.3 or JS not defined jsOrVmBlock: buffer[slice.a.. 0: - let found = c_memchr(s[start].unsafeAddr, cint(sub), cast[csize_t](length)) + let (sdata, _) = readRawData(s) + let found = c_memchr(addr sdata[start], cint(sub), cast[csize_t](length)) if not found.isNil: - return cast[int](found) -% cast[int](s.cstring) + return cast[int](found) -% cast[int](sdata) else: findImpl() @@ -2041,9 +2042,11 @@ func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, when declared(memmem): let subLen = sub.len if last < 0 and start < s.len and subLen != 0: - let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) + let (sdata, _) = readRawData(s) + let (subdata, _) = readRawData(sub) + let found = memmem(addr sdata[start], csize_t(s.len - start), subdata, csize_t(subLen)) result = if not found.isNil: - cast[int](found) -% cast[int](s.cstring) + cast[int](found) -% cast[int](sdata) else: -1 else: diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim index 767de111b5..44f745c264 100644 --- a/lib/std/formatfloat.nim +++ b/lib/std/formatfloat.nim @@ -19,7 +19,12 @@ proc addCstringN(result: var string, buf: cstring; buflen: int) = let oldLen = result.len let newLen = oldLen + buflen result.setLen newLen - c_memcpy(result[oldLen].addr, buf, buflen.csize_t) + {.cast(noSideEffect).}: + when declared(completeStore): + c_memcpy(beginStore(result, buflen, oldLen), buf, buflen.csize_t) + endStore(result) + else: + discard c_memcpy(result[oldLen].addr, buf, buflen.csize_t) import std/private/[dragonbox, schubfach] diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 73b28a68ba..8b6fc1b4e8 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -52,7 +52,7 @@ func addChars[T](result: var string, x: T, start: int, n: int) {.inline, enforce for i in 0.. 0 and line[last-1] == '\c': line.setLen(last-1) return last > 1 or fgetsSuccess @@ -565,8 +566,8 @@ proc readAllBuffer(file: File): string = result = "" var buffer = newString(BufSize) while true: - var bytesRead = readBuffer(file, addr(buffer[0]), BufSize) - when declared(completeStore): completeStore(buffer) + var bytesRead = readBuffer(file, beginStore(buffer, BufSize), BufSize) + endStore(buffer) if bytesRead == BufSize: result.add(buffer) else: @@ -592,8 +593,8 @@ proc readAllFile(file: File, len: int64): string = # We acquire the filesize beforehand and hope it doesn't change. # Speeds things up. result = newString(len) - let bytes = readBuffer(file, addr(result[0]), len) - when declared(completeStore): completeStore(result) + let bytes = readBuffer(file, beginStore(result, len.int), len.int) + endStore(result) if endOfFile(file): if bytes.int64 < len: result.setLen(bytes) diff --git a/lib/system.nim b/lib/system.nim index ecebd0c890..37fc344929 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1698,6 +1698,16 @@ when notJSnotNims and defined(nimSeqsV2): include "system/strs_v2" include "system/seqs_v2" +when not (notJSnotNims and defined(nimSeqsV2)): + # Fallback stubs for js/nimscript/non-V2 backends where strs_v2/v3 is not included. + # These are needed so that modules imported by system (e.g. syncio) can reference + # beginStore/endStore/readRawData without a 'when declared(...)' guard. + proc beginStore*(s: var string; ensuredLen: int; start = 0): ptr UncheckedArray[char] {.inline, noSideEffect.} = + result = cast[ptr UncheckedArray[char]](addr s[start]) + proc endStore*(s: var string) {.inline, noSideEffect.} = discard + template readRawData*(s: string): (ptr UncheckedArray[char], int) = + (cast[ptr UncheckedArray[char]](nil), s.len) + when not defined(js): template newSeqImpl(T, len) = result = newSeqOfCap[T](len) @@ -2928,7 +2938,9 @@ proc substr*(a: openArray[char]): string = result = newStringUninit(a.len) whenNotVmJsNims(): if a.len > 0: - copyMem(result[0].addr, a[0].unsafeAddr, a.len) + {.cast(noSideEffect).}: + copyMem(beginStore(result, a.len), a[0].unsafeAddr, a.len) + endStore(result) do: for i, ch in a: result[i] = ch @@ -2963,7 +2975,9 @@ proc substr*(s: string; first, last: int): string = # A bug with `magic: Slice` result = newStringUninit(L) whenNotVmJsNims(): if L > 0: - copyMem(result[0].addr, s[first].unsafeAddr, L) + let (src, _) = readRawData(s) + copyMem(beginStore(result, L), addr src[first], L) + endStore(result) do: for i in 0.. AlwaysAvail: compare inline prefix word (contains slen + chars 0-6) - if abytes != bbytes: return false - if slen <= PayloadSize: - let (la, pa) = a.guts + let aslen = ssLenOf(abytes) + let bslen = ssLenOf(bbytes) + if aslen <= AlwaysAvail and bslen <= AlwaysAvail: + return abytes == bbytes # SWAR: slen equal, data in bytes word + # Compute actual lengths (sentinels 254/255 → more.fullLen) + let la = if aslen > PayloadSize: a.more.fullLen else: aslen + let lb = if bslen > PayloadSize: b.more.fullLen else: bslen + if la != lb: return false + if la == 0: return true + if aslen <= PayloadSize and bslen <= PayloadSize: + # Both medium (slen == la == lb, so byte0 equal): compare prefix word + tail + if abytes != bbytes: return false + let (_, pa) = a.guts let (_, pb) = b.guts return cmpMem(addr pa[AlwaysAvail], addr pb[AlwaysAvail], la - AlwaysAvail) == 0 - # long: prefix matched; check lengths then compare the heap tail - let la = a.more.fullLen - if la != b.more.fullLen: return false - cmpMem(addr a.more.data[AlwaysAvail], addr b.more.data[AlwaysAvail], la - AlwaysAvail) == 0 + # At least one long (heap or static): delegate to cmpStringPtrs + cmpStringPtrs(unsafeAddr a, unsafeAddr b) == 0 proc continuesWith*(s, sub: SmallString; start: int): bool = if start < 0: return false @@ -672,6 +675,37 @@ proc completeStore(s: var SmallString) {.compilerproc, inline.} = proc completeStore*(s: var string) {.inline.} = completeStore(cast[ptr SmallString](addr s)[]) +proc beginStore*(s: var string; ensuredLen: int; start = 0): ptr UncheckedArray[char] {.inline, noSideEffect.} = + ## Prepares `s` for a bulk write of `ensuredLen` bytes starting at `start`. + ## The caller must ensure `s.len >= start + ensuredLen` (e.g. via `newString` or `setLen`). + ## Call `endStore(s)` afterwards to sync the inline cache. + {.cast(noSideEffect).}: + let ss = cast[ptr SmallString](addr s) + let slen = ssLen(ss[]) + if slen > PayloadSize: + ensureUniqueLong(ss[], ss[].more.fullLen, ss[].more.fullLen) + result = cast[ptr UncheckedArray[char]](addr ss[].more.data[start]) + else: + result = cast[ptr UncheckedArray[char]](cast[uint](inlinePtr(ss[])) + uint(start)) + +proc endStore*(s: var string) {.inline, noSideEffect.} = + ## Syncs the inline cache after bulk writes via `beginStore`. No-op for short/medium strings. + {.cast(noSideEffect).}: completeStore(cast[ptr SmallString](addr s)[]) + +proc rawDataImpl(ss: ptr SmallString): (ptr UncheckedArray[char], int) {.inline.} = + let slen = ssLen(ss[]) + let actualLen = if slen > PayloadSize: ss[].more.fullLen else: slen + let p = + if actualLen == 0: nil + elif slen > PayloadSize: cast[ptr UncheckedArray[char]](addr ss[].more.data[0]) + else: inlinePtr(ss[]) + (p, actualLen) + +template readRawData*(s: string): (ptr UncheckedArray[char], int) = + ## Returns `(dataPtr, length)` for read-only raw access to string data. + ## Template ensures no copy of `s` is made; ptr is valid while `s` is alive. + rawDataImpl(cast[ptr SmallString](unsafeAddr s)) + # These take `string` (tyString) so the codegen uses them directly, bypassing # strmantle.nim's versions which go through nimStrLen/nimStrAtMutV3 compilerproc calls. proc cmpStrings(a, b: string): int {.compilerproc, inline.} =