diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 8bab471ee7..6ffe561df7 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -344,7 +344,8 @@ proc expressionsNeedsTmp(p: BProc, a: TLoc): TLoc = proc genArgStringToCString(p: BProc, n: PNode; result: var Builder; needsTmp: bool) {.inline.} = var a = initLocExpr(p, n[0]) - let ra = withTmpIfNeeded(p, a, needsTmp).rdLoc + let tmp = withTmpIfNeeded(p, a, needsTmp) + let ra = if p.config.isDefined("nimsso"): addrLoc(p.config, tmp) else: tmp.rdLoc result.addCall(cgsymValue(p.module, "nimToCStringConv"), ra) proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; needsTmp = false) = diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 9f2ac2ff9b..9b8de6c4dd 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -320,12 +320,16 @@ proc genOpenArrayConv(p: BProc; d: TLoc; a: TLoc; flags: TAssignmentFlags) = p.s(cpsStmts).addCallStmt( cgsymValue(p.module, "nimPrepareStrMutationV2"), bra) - let rd = d.rdLoc - let ra = a.rdLoc - p.s(cpsStmts).addFieldAssignment(rd, "Field0", - cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil)) let la = lenExpr(p, a) + if p.config.isDefined("nimsso"): + let bra = byRefLoc(p, a) + p.s(cpsStmts).addFieldAssignment(rd, "Field0", + cCall(cgsymValue(p.module, "nimStrData"), bra)) + else: + let ra = a.rdLoc + p.s(cpsStmts).addFieldAssignment(rd, "Field0", + cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil)) p.s(cpsStmts).addFieldAssignment(rd, "Field1", la) else: internalError(p.config, a.lode.info, "cannot handle " & $a.t.kind) @@ -1316,8 +1320,13 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = let bra = byRefLoc(p, a) p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "nimPrepareStrMutationV2"), bra) - let ra = rdLoc(a) - putIntoDest(p, d, n, subscript(dataField(p, ra), rcb), a.storage) + if p.config.isDefined("nimsso") and ty.kind == tyString: + let bra = byRefLoc(p, a) + putIntoDest(p, d, n, + subscript(cCall(cgsymValue(p.module, "nimStrData"), bra), rcb), a.storage) + else: + let ra = rdLoc(a) + putIntoDest(p, d, n, subscript(dataField(p, ra), rcb), a.storage) proc genBracketExpr(p: BProc; n: PNode; d: var TLoc) = var ty = skipTypes(n[0].typ, abstractVarRange + tyUserTypeClasses) @@ -2124,12 +2133,20 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = let ra = rdLoc(a) putIntoDest(p, b, e, ra & cArgumentSeparator & ra & "Len_0", a.storage) of tyString, tySequence: - let ra = rdLoc(a) let la = lenExpr(p, a) - putIntoDest(p, b, e, - cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil) & - cArgumentSeparator & la, - a.storage) + if p.config.isDefined("nimsso") and + skipTypes(a.t, abstractVarRange).kind == tyString: + let bra = byRefLoc(p, a) + putIntoDest(p, b, e, + cCall(cgsymValue(p.module, "nimStrData"), bra) & + cArgumentSeparator & la, + a.storage) + else: + let ra = rdLoc(a) + putIntoDest(p, b, e, + cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil) & + cArgumentSeparator & la, + a.storage) of tyArray: let ra = rdLoc(a) let la = cIntValue(lengthOrd(p.config, a.t)) @@ -2710,9 +2727,9 @@ proc genConv(p: BProc, e: PNode, d: var TLoc) = proc convStrToCStr(p: BProc, n: PNode, d: var TLoc) = var a: TLoc = initLocExpr(p, n[0]) + let arg = if p.config.isDefined("nimsso"): addrLoc(p.config, a) else: rdLoc(a) putIntoDest(p, d, n, - cgCall(p, "nimToCStringConv", rdLoc(a)), -# "($1 ? $1->data : (NCSTRING)\"\")" % [a.rdLoc], + cgCall(p, "nimToCStringConv", arg), a.storage) proc convCStrToStr(p: BProc, n: PNode, d: var TLoc) = @@ -2789,13 +2806,19 @@ proc genMove(p: BProc; n: PNode; d: var TLoc) = var src: TLoc = initLocExpr(p, n[2]) let destVal = rdLoc(a) let srcVal = rdLoc(src) - p.s(cpsStmts).addSingleIfStmt( - cOp(NotEqual, - dotField(destVal, "p"), - dotField(srcVal, "p"))): + if p.config.isDefined("nimsso") and + n[1].typ.skipTypes(abstractVar).kind == tyString: + # SmallString: destroy dst then struct-copy src; no .p field aliasing needed genStmts(p, n[3]) - p.s(cpsStmts).addFieldAssignment(destVal, "len", dotField(srcVal, "len")) - p.s(cpsStmts).addFieldAssignment(destVal, "p", dotField(srcVal, "p")) + genAssignment(p, a, src, {}) + else: + p.s(cpsStmts).addSingleIfStmt( + cOp(NotEqual, + dotField(destVal, "p"), + dotField(srcVal, "p"))): + genStmts(p, n[3]) + p.s(cpsStmts).addFieldAssignment(destVal, "len", dotField(srcVal, "len")) + p.s(cpsStmts).addFieldAssignment(destVal, "p", dotField(srcVal, "p")) else: if d.k == locNone: d = getTemp(p, n.typ) if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}: @@ -2832,15 +2855,19 @@ proc genDestroy(p: BProc; n: PNode) = case t.kind of tyString: var a: TLoc = initLocExpr(p, arg) - let ra = rdLoc(a) - let rp = dotField(ra, "p") - p.s(cpsStmts).addSingleIfStmt( - cOp(And, rp, - cOp(Not, cOp(BitAnd, NimInt, - derefField(rp, "cap"), - NimStrlitFlag)))): - let fn = if optThreads in p.config.globalOptions: "deallocShared" else: "dealloc" - p.s(cpsStmts).addCallStmt(cgsymValue(p.module, fn), rp) + if p.config.isDefined("nimsso"): + # SmallString: delegate to nimDestroyStrV1 (rc-based, handles static strings) + p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "nimDestroyStrV1"), rdLoc(a)) + else: + let ra = rdLoc(a) + let rp = dotField(ra, "p") + p.s(cpsStmts).addSingleIfStmt( + cOp(And, rp, + cOp(Not, cOp(BitAnd, NimInt, + derefField(rp, "cap"), + NimStrlitFlag)))): + let fn = if optThreads in p.config.globalOptions: "deallocShared" else: "dealloc" + p.s(cpsStmts).addCallStmt(cgsymValue(p.module, fn), rp) of tySequence: var a: TLoc = initLocExpr(p, arg) let ra = rdLoc(a) @@ -4200,7 +4227,10 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; resul genConstObjConstr(p, n, isConst, result) of tyString, tyCstring: if optSeqDestructors in p.config.globalOptions and n.kind != nkNilLit and ty == tyString: - genStringLiteralV2Const(p.module, n, isConst, result) + if p.config.isDefined("nimsso"): + genStringLiteralV3Const(p.module, n, isConst, result) + else: + genStringLiteralV2Const(p.module, n, isConst, result) else: var d: TLoc = initLocExpr(p, n) result.add rdLoc(d) diff --git a/compiler/ccgliterals.nim b/compiler/ccgliterals.nim index a1ad3ae047..9913ba147e 100644 --- a/compiler/ccgliterals.nim +++ b/compiler/ccgliterals.nim @@ -22,7 +22,11 @@ template detectVersion(field, corename) = result = 1 proc detectStrVersion(m: BModule): int = - detectVersion(strVersion, "nimStrVersion") + if m.g.config.isDefined("nimsso") and + m.g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc, gcHooks}: + result = 3 + else: + detectVersion(strVersion, "nimStrVersion") proc detectSeqVersion(m: BModule): int = detectVersion(seqVersion, "nimSeqVersion") @@ -128,6 +132,155 @@ proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Bu result.addField(strInit, name = "p"): result.add(cCast(ptrType("NimStrPayload"), cAddr(pureLit))) +proc ssoCharLit(ch: char): string = + ## Return a C char literal for ch, with proper escaping. + const hexDigits = "0123456789abcdef" + result = "'" + case ch + of '\'': result.add("\\'") + of '\\': result.add("\\\\") + of '\0': result.add("\\0") + of '\n': result.add("\\n") + of '\r': result.add("\\r") + of '\t': result.add("\\t") + elif ch.ord < 32 or ch.ord == 127: + result.add("\\x") + result.add(hexDigits[ch.ord shr 4]) + result.add(hexDigits[ch.ord and 0xf]) + else: + result.add(ch) + result.add('\'') + +proc ssoPayloadLit(src: string; maxLen: int): string = + const AlwaysAvail = 7 + result = "{" + for i in 0.. 0: result.add(',') + let ch = if i < maxLen: src[i] else: '\0' + result.add(ssoCharLit(ch)) + result.add('}') + +proc genStringLiteralV3Const(m: BModule; n: PNode; isConst: bool; result: var Builder) = + # Inline SmallString struct initializer for use inside const aggregate types. + # Short strings (<=7 chars) embed all chars directly. Long strings reference + # a static LongString block emitted separately into cfsStrData. + const AlwaysAvail = 7 + let s = n.strVal + + cgsym(m, "SmallString") + cgsym(m, "LongString") + + var si: StructInitializer + result.addStructInitializer(si, kind = siOrderedStruct): + if s.len <= AlwaysAvail: + result.addField(si, name = "slen"): + result.addIntValue(s.len) + result.addField(si, name = "payload"): + result.add(ssoPayloadLit(s, s.len)) + result.addField(si, name = "more"): + result.add(NimNil) + else: + # Emit the LongString block into cfsStrData and reference it inline. + let dataName = getTempName(m) + var res = newBuilder("") + res.addVarWithTypeAndInitializer( + if isConst: AlwaysConst else: Global, + name = dataName): + res.addSimpleStruct(m, name = "", baseType = ""): + res.addField(name = "rc", typ = NimInt) + res.addField(name = "fullLen", typ = NimInt) + res.addField(name = "capImpl", typ = NimInt) + res.addArrayField(name = "data", elementType = NimChar, len = s.len + 1) + do: + var di: StructInitializer + res.addStructInitializer(di, kind = siOrderedStruct): + res.addField(di, name = "rc"): + res.addIntValue(1) + res.addField(di, name = "fullLen"): + res.addIntValue(s.len) + res.addField(di, name = "capImpl"): + res.addIntValue(0) # static, never freed + res.addField(di, name = "data"): + res.add(makeCString(s)) + m.s[cfsStrData].add(extract(res)) + result.addField(si, name = "slen"): + result.addIntValue(255) + result.addField(si, name = "payload"): + result.add(ssoPayloadLit(s, AlwaysAvail)) + result.addField(si, name = "more"): + result.add(cCast(ptrType("LongString"), cAddr(dataName))) + +# ------ Version 3: SmallString (SSO) strings -------------------------------- + +proc genStringLiteralV3(m: BModule; n: PNode; isConst: bool; result: var Builder) = + # SmallString literal. Always generate a fresh SmallString variable (like v2 + # always generates a fresh outer NimStringV2). For long strings, cache the + # LongString payload to avoid duplicates within a module. + const AlwaysAvail = 7 # must match strs_v3.nim + let s = n.strVal + let tmp = getTempName(m) + result.add tmp + + cgsym(m, "SmallString") + cgsym(m, "LongString") + + var res = newBuilder("") + if s.len <= AlwaysAvail: + # Short: all chars fit in payload, more = NULL. + res.addVarWithInitializer( + if isConst: AlwaysConst else: Global, + name = tmp, typ = "SmallString"): + var si: StructInitializer + res.addStructInitializer(si, kind = siOrderedStruct): + res.addField(si, name = "slen"): + res.addIntValue(s.len) + res.addField(si, name = "payload"): + res.add(ssoPayloadLit(s, s.len)) + res.addField(si, name = "more"): + res.add(NimNil) + else: + # Long: cache the LongString block to emit it only once per module per string. + # Always generate a fresh SmallString pointing at the (possibly cached) block. + let id = nodeTableTestOrSet(m.dataCache, n, m.labels) + var dataName: string + if id == m.labels: + dataName = getTempName(m) + res.addVarWithTypeAndInitializer( + if isConst: AlwaysConst else: Global, + name = dataName): + res.addSimpleStruct(m, name = "", baseType = ""): + res.addField(name = "rc", typ = NimInt) + res.addField(name = "fullLen", typ = NimInt) + res.addField(name = "capImpl", typ = NimInt) + res.addArrayField(name = "data", elementType = NimChar, len = s.len + 1) + do: + var di: StructInitializer + res.addStructInitializer(di, kind = siOrderedStruct): + res.addField(di, name = "rc"): + res.addIntValue(1) + res.addField(di, name = "fullLen"): + res.addIntValue(s.len) + res.addField(di, name = "capImpl"): + res.addIntValue(0) # bit 0 = 0: static, never freed + res.addField(di, name = "data"): + res.add(makeCString(s)) + else: + dataName = m.tmpBase & $id + # PayloadSize = AlwaysAvail + sizeof(pointer) - 1; sentinel slen = PayloadSize+1 + # We just use a large value (255) that is guaranteed > PayloadSize on all platforms. + res.addVarWithInitializer( + if isConst: AlwaysConst else: Global, + name = tmp, typ = "SmallString"): + var si: StructInitializer + res.addStructInitializer(si, kind = siOrderedStruct): + res.addField(si, name = "slen"): + res.addIntValue(255) # > PayloadSize on all platforms => long sentinel + res.addField(si, name = "payload"): + res.add(ssoPayloadLit(s, AlwaysAvail)) + res.addField(si, name = "more"): + res.add(cCast(ptrType("LongString"), cAddr(dataName))) + m.s[cfsStrData].add(extract(res)) + # ------ Version selector --------------------------------------------------- proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo; @@ -138,6 +291,8 @@ proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo; let tmp = getTempName(m) genStringLiteralDataOnlyV2(m, s, tmp, isConst) result.add tmp + of 3: + localError(m.config, info, "genStringLiteralDataOnly not supported for SmallString (nimsso)") else: localError(m.config, info, "cannot determine how to produce code for string literal") @@ -148,5 +303,6 @@ proc genStringLiteral(m: BModule; n: PNode; result: var Builder) = case detectStrVersion(m) of 0, 1: genStringLiteralV1(m, n, result) of 2: genStringLiteralV2(m, n, isConst = true, result) + of 3: genStringLiteralV3(m, n, isConst = true, result) else: localError(m.config, n.info, "cannot determine how to produce code for string literal") diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 67b7469cb0..97839fcb38 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -339,6 +339,10 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope = cgsym(m, "NimStrPayload") cgsym(m, "NimStringV2") result = typeNameOrLiteral(m, typ, "NimStringV2") + of 3: + cgsym(m, "LongString") + cgsym(m, "SmallString") + result = typeNameOrLiteral(m, typ, "SmallString") else: cgsym(m, "NimStringDesc") result = typeNameOrLiteral(m, typ, "NimStringDesc*") diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 5339b5e680..d7054ceec9 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -389,7 +389,11 @@ proc lenField(p: BProc, val: Rope): Rope {.inline.} = proc lenExpr(p: BProc; a: TLoc): Rope = if optSeqDestructors in p.config.globalOptions: - result = dotField(rdLoc(a), "len") + if p.config.isDefined("nimsso") and a.t != nil and + a.t.skipTypes(abstractInst).kind == tyString: + result = cCall(cgsymValue(p.module, "nimStrLen"), rdLoc(a)) + else: + result = dotField(rdLoc(a), "len") else: let ra = rdLoc(a) result = cIfExpr(ra, lenField(p, ra), cIntValue(0)) @@ -530,7 +534,15 @@ proc resetLoc(p: BProc, loc: var TLoc) = let atyp = skipTypes(loc.t, abstractInst) let rl = rdLoc(loc) - if atyp.kind in {tyVar, tyLent}: + if typ.kind == tyString and p.config.isDefined("nimsso"): + # SmallString zero state: slen=0 suffices (slen<=AlwaysAvail => inline, no heap) + if atyp.kind in {tyVar, tyLent}: + p.s(cpsStmts).addAssignment(derefField(rl, "slen"), cIntValue(0)) + p.s(cpsStmts).addAssignment(derefField(rl, "more"), NimNil) + else: + p.s(cpsStmts).addAssignment(dotField(rl, "slen"), cIntValue(0)) + p.s(cpsStmts).addAssignment(dotField(rl, "more"), NimNil) + elif atyp.kind in {tyVar, tyLent}: p.s(cpsStmts).addAssignment(derefField(rl, "len"), cIntValue(0)) p.s(cpsStmts).addAssignment(derefField(rl, "p"), NimNil) else: @@ -580,8 +592,13 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = let typ = loc.t if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}: let rl = rdLoc(loc) - p.s(cpsStmts).addFieldAssignment(rl, "len", cIntValue(0)) - p.s(cpsStmts).addFieldAssignment(rl, "p", NimNil) + if skipTypes(typ, abstractInst + {tyStatic}).kind == tyString and p.config.isDefined("nimsso"): + # SmallString zero state: slen=0 suffices + p.s(cpsStmts).addFieldAssignment(rl, "slen", cIntValue(0)) + p.s(cpsStmts).addFieldAssignment(rl, "more", NimNil) + else: + p.s(cpsStmts).addFieldAssignment(rl, "len", cIntValue(0)) + p.s(cpsStmts).addFieldAssignment(rl, "p", NimNil) elif not isComplexValueType(typ): if containsGarbageCollectedRef(loc.t): var nilLoc: TLoc = initLoc(locTemp, loc.lode, OnStack) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index f0a5acc78c..e05b14d460 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -701,11 +701,18 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedAsgn, attachedDeepCopy, attachedDup: body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c, x), y) of attachedSink: - let moveCall = genBuiltin(c, mMove, "move", x) - moveCall.add y - doAssert t.destructor != nil - moveCall.add destructorCall(c, t.destructor, x) - body.add moveCall + if c.g.config.isDefined("nimsso"): + # SmallString: destroy old dst, then bit-copy src (no rc increment — this is a move). + # No .p aliasing check needed; rc-based destroy handles COW sharing correctly. + doAssert t.destructor != nil + body.add destructorCall(c, t.destructor, x) + body.add newAsgnStmt(x, y) + else: + let moveCall = genBuiltin(c, mMove, "move", x) + moveCall.add y + doAssert t.destructor != nil + moveCall.add destructorCall(c, t.destructor, x) + body.add moveCall of attachedDestructor: body.add genBuiltin(c, mDestroy, "destroy", x) of attachedTrace: diff --git a/lib/system.nim b/lib/system.nim index 306818ffa0..19875d778e 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1641,7 +1641,7 @@ when defined(windows): const ERROR_BAD_EXE_FORMAT = 193 when notJSnotNims: - when defined(nimSeqsV2): + when defined(nimSeqsV2) and not defined(nimsso): proc nimToCStringConv(s: NimStringV2): cstring {.compilerproc, nonReloadable, inline.} when hostOS != "standalone" and hostOS != "any": @@ -1689,7 +1689,10 @@ when not defined(nimIcIntegrityChecks): export exceptions when notJSnotNims and defined(nimSeqsV2): - include "system/strs_v2" + when defined(nimsso): + include "system/strs_v3" + else: + include "system/strs_v2" include "system/seqs_v2" when not defined(js): diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 0955222ec1..422b78f76e 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -62,9 +62,14 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = case mt.kind of tyString: when defined(nimSeqsV2): - var x = cast[ptr NimStringV2](dest) - var s2 = cast[ptr NimStringV2](s)[] - nimAsgnStrV2(x[], s2) + when defined(nimsso): + var x = cast[ptr SmallString](dest) + var s2 = cast[ptr SmallString](s)[] + nimAsgnStrV2(x[], s2) + else: + var x = cast[ptr NimStringV2](dest) + var s2 = cast[ptr NimStringV2](s)[] + nimAsgnStrV2(x[], s2) else: var x = cast[PPointer](dest) var s2 = cast[PPointer](s)[] @@ -245,8 +250,11 @@ proc genericReset(dest: pointer, mt: PNimType) = unsureAsgnRef(cast[PPointer](dest), nil) of tyString: when defined(nimSeqsV2): - var s = cast[ptr NimStringV2](dest) - frees(s[]) + when defined(nimsso): + nimDestroyStrV1(cast[ptr SmallString](dest)[]) + else: + var s = cast[ptr NimStringV2](dest) + frees(s[]) zeroMem(dest, mt.size) else: unsureAsgnRef(cast[PPointer](dest), nil) diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index fdf1499e5f..06100b3611 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -92,9 +92,14 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = case mt.kind of tyString: when defined(nimSeqsV2): - var x = cast[ptr NimStringV2](dest) - var s2 = cast[ptr NimStringV2](s)[] - nimAsgnStrV2(x[], s2) + when defined(nimsso): + var x = cast[ptr SmallString](dest) + var s2 = cast[ptr SmallString](s)[] + nimAsgnStrV2(x[], s2) + else: + var x = cast[ptr NimStringV2](dest) + var s2 = cast[ptr NimStringV2](s)[] + nimAsgnStrV2(x[], s2) else: var x = cast[PPointer](dest) var s2 = cast[PPointer](s)[] diff --git a/lib/system/strs_v3.nim b/lib/system/strs_v3.nim new file mode 100644 index 0000000000..53db3117be --- /dev/null +++ b/lib/system/strs_v3.nim @@ -0,0 +1,495 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2026 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Small String Optimization (SSO) implementation used by Nim's core. + +const + AlwaysAvail = 7 + PayloadSize = AlwaysAvail + sizeof(pointer) - 1 # -1 reserves the last byte for '\0' + +proc atomicAddFetch(p: var int; v: int): int {.importc: "__sync_add_and_fetch", nodecl.} +proc atomicSubFetch(p: var int; v: int): int {.importc: "__sync_sub_and_fetch", nodecl.} + +type + LongString {.core.} = object + rc: int # atomic reference count; 1 = unique owner + fullLen: int + capImpl: int # bit 0: heap-allocated; upper bits: capacity (cap = capImpl shr 1) + data: UncheckedArray[char] + + SmallString {.core.} = object + slen: byte # when > PayloadSize, `more` is valid ptr + payload: array[AlwaysAvail, char] + more: ptr LongString # when long: pointer; when small (len 8..15): bytes 7..14 stored here + +proc resize(old: int): int {.inline.} = + ## Capacity growth factor shared with seqs_v2.nim. + if old <= 0: result = 4 + elif old <= high(int16): result = old * 2 + else: result = old div 2 + old + +proc `=destroy`*(s: var SmallString) = + if int(s.slen) > PayloadSize and (s.more.capImpl and 1) == 1: + if atomicSubFetch(s.more.rc, 1) == 0: + dealloc(s.more) + +proc `=wasMoved`*(s: var SmallString) {.inline.} = + s.slen = 0 + +proc `=sink`*(dst: var SmallString; src: SmallString) = + `=destroy`(dst) + copyMem(addr dst, unsafeAddr src, sizeof(SmallString)) + +proc `=copy`*(dst: var SmallString; src: SmallString) = + if int(src.slen) <= PayloadSize: + `=destroy`(dst) # dst may have been a long string + copyMem(addr dst, unsafeAddr src, sizeof(SmallString)) + else: + if addr(dst) == unsafeAddr(src): return + `=destroy`(dst) + # COW: share the block, bump refcount — no allocation needed + if (src.more.capImpl and 1) == 1: + discard atomicAddFetch(src.more.rc, 1) + copyMem(addr dst, unsafeAddr src, sizeof(SmallString)) + +proc `=dup`*(src: SmallString): SmallString = + copyMem(addr result, unsafeAddr src, sizeof(SmallString)) + if int(src.slen) > PayloadSize and (src.more.capImpl and 1) == 1: + discard atomicAddFetch(src.more.rc, 1) + +proc ensureUniqueLong(s: var SmallString; oldLen, newLen: int) = + # Ensure s.more is a unique (rc=1) heap block with capacity >= newLen, preserving existing data. + # s must already be a long string on entry. + let heapAlloc = (s.more.capImpl and 1) == 1 + let unique = heapAlloc and s.more.rc == 1 + let cap = s.more.capImpl shr 1 + if unique and newLen <= cap: + s.more.fullLen = newLen + else: + let newCap = max(newLen, oldLen * 2) + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + newCap + 1)) + p.rc = 1 + p.fullLen = newLen + p.capImpl = (newCap shl 1) or 1 + let old = s.more + copyMem(addr p.data[0], addr old.data[0], oldLen + 1) # +1 preserves the '\0' + if heapAlloc and atomicSubFetch(old.rc, 1) == 0: + dealloc(old) + s.more = p + +proc len*(s: SmallString): int {.inline.} = + result = int s.slen + if result > PayloadSize: + result = s.more.fullLen + +template guts(s: SmallString): (int, ptr UncheckedArray[char]) = + let slen = int s.slen + if slen > PayloadSize: + (s.more.fullLen, cast[ptr UncheckedArray[char]](addr s.more.data[0])) + else: + (slen, cast[ptr UncheckedArray[char]](addr s.payload[0])) + +proc `[]`*(s: SmallString; i: int): char {.inline.} = + let slen = int s.slen + if slen <= PayloadSize: + # unchecked: when i >= 7 we store into the `more` overlay + result = (cast[ptr UncheckedArray[char]](addr s.payload[0]))[i] + elif i < AlwaysAvail: + result = s.payload[i] + else: + result = s.more.data[i] + +proc `[]=`*(s: var SmallString; i: int; c: char) = + let slen = int s.slen + if slen <= PayloadSize: + # unchecked: when i >= 7 we store into the `more` overlay + (cast[ptr UncheckedArray[char]](addr s.payload[0]))[i] = c + else: + let l = s.more.fullLen + ensureUniqueLong(s, l, l) # COW if shared; length unchanged + s.more.data[i] = c + if i < AlwaysAvail: + s.payload[i] = c + +proc cmp*(a, b: SmallString): int = + # Use slen directly for prefix length: for short/medium it is the real length, + # for long it is the sentinel (> AlwaysAvail), so min(..., AlwaysAvail) still gives 7. + # This avoids dereferencing `more` before the prefix comparison. + let pfxLen = min(min(int a.slen, int b.slen), AlwaysAvail) + result = cmpMem(unsafeAddr a.payload[0], unsafeAddr b.payload[0], pfxLen) + if result != 0: return + # Prefix matched — now fetch actual lengths (dereferences `more` only if long) + let la = if int(a.slen) > PayloadSize: a.more.fullLen else: int(a.slen) + let lb = if int(b.slen) > PayloadSize: b.more.fullLen else: int(b.slen) + let minLen = min(la, lb) + if minLen <= AlwaysAvail: + result = la - lb + return + let (_, pa) = a.guts + let (_, pb) = b.guts + result = cmpMem(addr pa[AlwaysAvail], addr pb[AlwaysAvail], minLen - AlwaysAvail) + if result == 0: + result = la - lb + +proc `==`*(a, b: SmallString): bool = + if a.slen != b.slen: return false + # slen equal: for short/medium this means equal lengths; for long (both sentinel) we still need fullLen. + let slen = int(a.slen) + let pfxLen = min(slen, AlwaysAvail) + if cmpMem(unsafeAddr a.payload[0], unsafeAddr b.payload[0], pfxLen) != 0: return false + if slen <= AlwaysAvail: return true + if slen <= PayloadSize: + # medium: guts gives the UncheckedArray without a heap dereference + let (la, pa) = a.guts + let (_, pb) = b.guts + return cmpMem(addr pa[pfxLen], addr pb[pfxLen], la - pfxLen) == 0 + # long: fetch actual lengths only after prefix matched + let la = a.more.fullLen + if la != b.more.fullLen: return false + cmpMem(addr a.more.data[pfxLen], addr b.more.data[pfxLen], la - pfxLen) == 0 + +proc `<=`*(a, b: SmallString): bool {.inline.} = cmp(a, b) <= 0 + +proc continuesWith*(s, sub: SmallString; start: int): bool = + if start < 0: return false + let subslen = int(sub.slen) + if subslen == 0: return true + # Compare inline prefix first — no `more` dereference yet. + # For long sub, subslen is the sentinel (> AlwaysAvail), so pfxLen is capped correctly. + let pfxLen = min(subslen, max(0, AlwaysAvail - start)) + if pfxLen > 0: + if cmpMem(unsafeAddr s.payload[start], unsafeAddr sub.payload[0], pfxLen) != 0: + return false + # Prefix matched (or start >= AlwaysAvail); now fetch actual lengths + let subLen = if subslen > PayloadSize: sub.more.fullLen else: subslen + let sLen = if int(s.slen) > PayloadSize: s.more.fullLen else: int(s.slen) + if start + subLen > sLen: return false + if pfxLen == subLen: return true # sub fully compared within the prefix + let (_, sp) = s.guts + let (_, subp) = sub.guts + cmpMem(addr sp[start + pfxLen], addr subp[pfxLen], subLen - pfxLen) == 0 + +proc startsWith*(s, sub: SmallString): bool {.inline.} = continuesWith(s, sub, 0) +proc endsWith*(s, sub: SmallString): bool {.inline.} = continuesWith(s, sub, s.len - sub.len) + + +proc add*(s: var SmallString; c: char) = + let slen = int(s.slen) + if slen <= PayloadSize: + let newLen = slen + 1 + if newLen <= PayloadSize: + let inl = cast[ptr UncheckedArray[char]](addr s.payload[0]) + inl[slen] = c + inl[newLen] = '\0' + s.slen = byte(newLen) + else: + # transition from medium (slen == PayloadSize) to long + let cap = newLen * 2 + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + cap + 1)) + p.rc = 1 + p.fullLen = newLen + p.capImpl = (cap shl 1) or 1 + copyMem(addr p.data[0], cast[ptr UncheckedArray[char]](addr s.payload[0]), slen) + p.data[slen] = c + p.data[newLen] = '\0' + # payload[0..AlwaysAvail-1] already correct; slen >= AlwaysAvail so no update needed + s.more = p + s.slen = byte(PayloadSize + 1) + else: + let l = s.more.fullLen # fetch fullLen only in the long path + ensureUniqueLong(s, l, l + 1) + s.more.data[l] = c + s.more.data[l + 1] = '\0' + # l >= PayloadSize > AlwaysAvail, so prefix is unaffected + +proc add*(s: var SmallString; t: SmallString) = + let slen = int(s.slen) + let (tl, tp) = t.guts # fetch t's guts before any mutation (aliasing safety) + if tl == 0: return + if slen <= PayloadSize: + let sl = slen # for short/medium, slen IS the actual length + let newLen = sl + tl + if newLen <= PayloadSize: + let inl = cast[ptr UncheckedArray[char]](addr s.payload[0]) + copyMem(addr inl[sl], tp, tl) + inl[newLen] = '\0' + s.slen = byte(newLen) + else: + # transition to long + let cap = newLen * 2 + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + cap + 1)) + p.rc = 1 + p.fullLen = newLen + p.capImpl = (cap shl 1) or 1 + copyMem(addr p.data[0], cast[ptr UncheckedArray[char]](addr s.payload[0]), sl) + copyMem(addr p.data[sl], tp, tl) + p.data[newLen] = '\0' + # update prefix bytes that come from t (only when sl < AlwaysAvail) + if sl < AlwaysAvail: + copyMem(addr s.payload[sl], tp, min(AlwaysAvail - sl, tl)) + s.more = p + s.slen = byte(PayloadSize + 1) + else: + let sl = s.more.fullLen # fetch fullLen only in the long path + let newLen = sl + tl + # tp was read before ensureUniqueLong: if t.more == s.more, rc decrements but won't hit 0 + ensureUniqueLong(s, sl, newLen) + copyMem(addr s.more.data[sl], tp, tl) + s.more.data[newLen] = '\0' + # sl >= PayloadSize > AlwaysAvail, so prefix is unaffected + +proc `&`*(a, b: SmallString): SmallString = + result = a + result.add(b) + +proc toSmallString*(s: openArray[char]): SmallString = + let l = s.len + if l == 0: return + if l <= PayloadSize: + result.slen = byte(l) + let inl = cast[ptr UncheckedArray[char]](addr result.payload[0]) + copyMem(inl, unsafeAddr s[0], l) + inl[l] = '\0' + else: + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + l + 1)) + p.rc = 1 + p.fullLen = l + p.capImpl = (l shl 1) or 1 + copyMem(addr p.data[0], unsafeAddr s[0], l) + p.data[l] = '\0' + copyMem(addr result.payload[0], unsafeAddr s[0], AlwaysAvail) + result.slen = byte(PayloadSize + 1) + result.more = p + +{.push overflowChecks: off, rangeChecks: off.} + +proc prepareAddLong(s: var SmallString; newLen: int) = + # Reserve capacity for newLen in the long-string block without changing logical length. + let heapAlloc = (s.more.capImpl and 1) == 1 + let cap = s.more.capImpl shr 1 + if heapAlloc and s.more.rc == 1 and newLen <= cap: + discard # already unique with sufficient capacity + else: + let oldLen = s.more.fullLen + let newCap = max(newLen, oldLen * 2) + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + newCap + 1)) + p.rc = 1 + p.fullLen = oldLen # logical length unchanged — caller sets it after writing data + p.capImpl = (newCap shl 1) or 1 + let old = s.more + copyMem(addr p.data[0], addr old.data[0], oldLen + 1) + if heapAlloc and atomicSubFetch(old.rc, 1) == 0: + dealloc(old) + s.more = p + +proc prepareAdd*(s: var SmallString; addLen: int) {.compilerRtl.} = + ## Ensure s has room for addLen more characters without changing its length. + let slen = int(s.slen) + let curLen = if slen > PayloadSize: s.more.fullLen else: slen + let newLen = curLen + addLen + if slen <= PayloadSize: + if newLen > PayloadSize: + # transition to long: allocate, copy existing data + let newCap = newLen * 2 + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + newCap + 1)) + p.rc = 1 + p.fullLen = curLen + p.capImpl = (newCap shl 1) or 1 + let inl = cast[ptr UncheckedArray[char]](addr s.payload[0]) + copyMem(addr p.data[0], inl, curLen + 1) + s.more = p + s.slen = byte(PayloadSize + 1) + # else: short/medium — inline capacity always sufficient (struct is fixed size) + else: + prepareAddLong(s, newLen) + +proc nimAddCharV1*(s: var SmallString; c: char) {.compilerRtl, inline.} = + prepareAdd(s, 1) + s.add(c) + +proc toNimStr*(str: cstring; len: int): SmallString {.compilerproc.} = + if len <= 0: return + if len <= PayloadSize: + result.slen = byte(len) + let inl = cast[ptr UncheckedArray[char]](addr result.payload[0]) + copyMem(inl, str, len) + inl[len] = '\0' + else: + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + len + 1)) + p.rc = 1 + p.fullLen = len + p.capImpl = (len shl 1) or 1 + copyMem(addr p.data[0], str, len) + p.data[len] = '\0' + copyMem(addr result.payload[0], str, AlwaysAvail) + result.slen = byte(PayloadSize + 1) + result.more = p + +proc cstrToNimstr*(str: cstring): SmallString {.compilerRtl.} = + if str == nil: return + toNimStr(str, str.len) + +proc nimToCStringConv*(s: var SmallString): cstring {.compilerproc, nonReloadable, inline.} = + ## Returns a null-terminated C string pointer into s's data. + ## Takes by var (pointer) so addr s.payload[0] is always into the caller's SmallString. + if int(s.slen) > PayloadSize: + cast[cstring](addr s.more.data[0]) + else: + cast[cstring](addr s.payload[0]) + +proc appendString*(dest: var SmallString; src: SmallString) {.compilerproc, inline.} = + dest.add(src) + +proc appendChar*(dest: var SmallString; c: char) {.compilerproc, inline.} = + dest.add(c) + +proc rawNewString*(space: int): SmallString {.compilerproc.} = + ## Returns an empty SmallString with capacity reserved for `space` chars (newStringOfCap). + if space <= 0: return + if space <= PayloadSize: + discard # inline capacity is always available; nothing to pre-allocate + else: + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + space + 1)) + p.rc = 1 + p.fullLen = 0 + p.capImpl = (space shl 1) or 1 + p.data[0] = '\0' + result.more = p + result.slen = byte(PayloadSize + 1) + +proc mnewString*(len: int): SmallString {.compilerproc.} = + ## Returns a SmallString of `len` zero characters (newString). + if len <= 0: return + if len <= PayloadSize: + result.slen = byte(len) + # payload is zero-initialized by default (result is zero) + cast[ptr UncheckedArray[char]](addr result.payload[0])[len] = '\0' + else: + let p = cast[ptr LongString](alloc0(sizeof(int) * 3 + len + 1)) + p.rc = 1 + p.fullLen = len + p.capImpl = (len shl 1) or 1 + # data is zeroed by alloc0; data[len] is '\0' too + result.more = p + result.slen = byte(PayloadSize + 1) + +proc setLengthStrV2*(s: var SmallString; newLen: int) {.compilerRtl.} = + ## Sets the length of s to newLen, zeroing new bytes on growth. + let slen = int(s.slen) + let curLen = if slen > PayloadSize: s.more.fullLen else: slen + if newLen == curLen: return + if newLen <= 0: + if slen > PayloadSize: + if (s.more.capImpl and 1) == 1 and s.more.rc == 1: + s.more.fullLen = 0 + s.more.data[0] = '\0' + else: + # shared block: detach and go back to empty inline + `=destroy`(s) + s.slen = 0 + else: + s.slen = 0 + s.payload[0] = '\0' + return + if slen <= PayloadSize: + if newLen <= PayloadSize: + if newLen > curLen: + let inl = cast[ptr UncheckedArray[char]](addr s.payload[0]) + zeroMem(addr inl[curLen], newLen - curLen) + inl[newLen] = '\0' + else: + cast[ptr UncheckedArray[char]](addr s.payload[0])[newLen] = '\0' + s.slen = byte(newLen) + else: + # grow into long + let newCap = newLen * 2 + let p = cast[ptr LongString](alloc0(sizeof(int) * 3 + newCap + 1)) + p.rc = 1 + p.fullLen = newLen + p.capImpl = (newCap shl 1) or 1 + copyMem(addr p.data[0], cast[ptr UncheckedArray[char]](addr s.payload[0]), curLen) + # bytes [curLen..newLen] zeroed by alloc0; p.data[newLen] = '\0' by alloc0 + s.more = p + s.slen = byte(PayloadSize + 1) + else: + # currently long + if newLen <= PayloadSize: + # shrink back to inline + let old = s.more + let heapAlloc = (old.capImpl and 1) == 1 + let inl = cast[ptr UncheckedArray[char]](addr s.payload[0]) + copyMem(inl, addr old.data[0], newLen) + inl[newLen] = '\0' + if heapAlloc and atomicSubFetch(old.rc, 1) == 0: + dealloc(old) + s.slen = byte(newLen) + else: + ensureUniqueLong(s, curLen, newLen) + if newLen > curLen: + zeroMem(addr s.more.data[curLen], newLen - curLen) + s.more.data[newLen] = '\0' + s.more.fullLen = newLen + +proc nimAsgnStrV2*(a: var SmallString; b: SmallString) {.compilerRtl.} = + `=copy`(a, b) + +proc nimPrepareStrMutationImpl(s: var SmallString) = + # Called when s holds a static (non-heap) LongString block. COW: allocate a fresh copy. + let old = s.more + let oldLen = old.fullLen + let p = cast[ptr LongString](alloc(sizeof(int) * 3 + oldLen + 1)) + p.rc = 1 + p.fullLen = oldLen + p.capImpl = (oldLen shl 1) or 1 + copyMem(addr p.data[0], addr old.data[0], oldLen + 1) + s.more = p + +proc nimPrepareStrMutationV2*(s: var SmallString) {.compilerRtl, inline.} = + if int(s.slen) > PayloadSize and (s.more.capImpl and 1) == 0: + nimPrepareStrMutationImpl(s) + +proc prepareMutation*(s: var string) {.inline.} = + {.cast(noSideEffect).}: + nimPrepareStrMutationV2(cast[ptr SmallString](addr s)[]) + +proc nimAddStrV1*(s: var SmallString; src: SmallString) {.compilerRtl, inline.} = + s.add(src) + +proc nimDestroyStrV1*(s: SmallString) {.compilerRtl, inline.} = + if int(s.slen) > PayloadSize and (s.more.capImpl and 1) == 1: + if atomicSubFetch(s.more.rc, 1) == 0: + dealloc(s.more) + +proc nimStrAtLe*(s: SmallString; idx: int; ch: char): bool {.compilerRtl, inline.} = + let l = s.len + result = idx < l and s[idx] <= ch + +func capacity*(self: SmallString): int {.inline.} = + ## Returns the current capacity of the string. + let slen = int(self.slen) + if slen > PayloadSize: + self.more.capImpl shr 1 + else: + PayloadSize + +proc nimStrLen*(s: SmallString): int {.compilerproc, inline.} = + ## Returns the length of s. Called by the codegen for `mLen` on strings with -d:nimsso. + s.len + +proc nimStrData*(s: var SmallString): ptr UncheckedArray[char] {.compilerproc, inline.} = + ## Returns a pointer to the char data of s. Called by codegen for subscript and slice with -d:nimsso. + let slen = int(s.slen) + if slen > PayloadSize: cast[ptr UncheckedArray[char]](addr s.more.data[0]) + else: cast[ptr UncheckedArray[char]](addr s.payload[0]) + +proc eqStrings*(a, b: SmallString): bool {.compilerproc, inline.} = a == b + +proc cmpStrings*(a, b: SmallString): int {.compilerproc, inline.} = cmp(a, b) + +{.pop.}