WIP: SSO based strings for ARC/ORC, opt-in via -d:nimsso

This commit is contained in:
araq
2026-03-07 17:10:55 +01:00
parent 7a87e7d199
commit 2e4ea76fc4
10 changed files with 776 additions and 50 deletions

View File

@@ -344,7 +344,8 @@ proc expressionsNeedsTmp(p: BProc, a: TLoc): TLoc =
proc genArgStringToCString(p: BProc, n: PNode; result: var Builder; needsTmp: bool) {.inline.} =
var a = initLocExpr(p, n[0])
let ra = withTmpIfNeeded(p, a, needsTmp).rdLoc
let tmp = withTmpIfNeeded(p, a, needsTmp)
let ra = if p.config.isDefined("nimsso"): addrLoc(p.config, tmp) else: tmp.rdLoc
result.addCall(cgsymValue(p.module, "nimToCStringConv"), ra)
proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; needsTmp = false) =

View File

@@ -320,12 +320,16 @@ proc genOpenArrayConv(p: BProc; d: TLoc; a: TLoc; flags: TAssignmentFlags) =
p.s(cpsStmts).addCallStmt(
cgsymValue(p.module, "nimPrepareStrMutationV2"),
bra)
let rd = d.rdLoc
let ra = a.rdLoc
p.s(cpsStmts).addFieldAssignment(rd, "Field0",
cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil))
let la = lenExpr(p, a)
if p.config.isDefined("nimsso"):
let bra = byRefLoc(p, a)
p.s(cpsStmts).addFieldAssignment(rd, "Field0",
cCall(cgsymValue(p.module, "nimStrData"), bra))
else:
let ra = a.rdLoc
p.s(cpsStmts).addFieldAssignment(rd, "Field0",
cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil))
p.s(cpsStmts).addFieldAssignment(rd, "Field1", la)
else:
internalError(p.config, a.lode.info, "cannot handle " & $a.t.kind)
@@ -1316,8 +1320,13 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) =
let bra = byRefLoc(p, a)
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "nimPrepareStrMutationV2"),
bra)
let ra = rdLoc(a)
putIntoDest(p, d, n, subscript(dataField(p, ra), rcb), a.storage)
if p.config.isDefined("nimsso") and ty.kind == tyString:
let bra = byRefLoc(p, a)
putIntoDest(p, d, n,
subscript(cCall(cgsymValue(p.module, "nimStrData"), bra), rcb), a.storage)
else:
let ra = rdLoc(a)
putIntoDest(p, d, n, subscript(dataField(p, ra), rcb), a.storage)
proc genBracketExpr(p: BProc; n: PNode; d: var TLoc) =
var ty = skipTypes(n[0].typ, abstractVarRange + tyUserTypeClasses)
@@ -2124,12 +2133,20 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
let ra = rdLoc(a)
putIntoDest(p, b, e, ra & cArgumentSeparator & ra & "Len_0", a.storage)
of tyString, tySequence:
let ra = rdLoc(a)
let la = lenExpr(p, a)
putIntoDest(p, b, e,
cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil) &
cArgumentSeparator & la,
a.storage)
if p.config.isDefined("nimsso") and
skipTypes(a.t, abstractVarRange).kind == tyString:
let bra = byRefLoc(p, a)
putIntoDest(p, b, e,
cCall(cgsymValue(p.module, "nimStrData"), bra) &
cArgumentSeparator & la,
a.storage)
else:
let ra = rdLoc(a)
putIntoDest(p, b, e,
cIfExpr(dataFieldAccessor(p, ra), dataField(p, ra), NimNil) &
cArgumentSeparator & la,
a.storage)
of tyArray:
let ra = rdLoc(a)
let la = cIntValue(lengthOrd(p.config, a.t))
@@ -2710,9 +2727,9 @@ proc genConv(p: BProc, e: PNode, d: var TLoc) =
proc convStrToCStr(p: BProc, n: PNode, d: var TLoc) =
var a: TLoc = initLocExpr(p, n[0])
let arg = if p.config.isDefined("nimsso"): addrLoc(p.config, a) else: rdLoc(a)
putIntoDest(p, d, n,
cgCall(p, "nimToCStringConv", rdLoc(a)),
# "($1 ? $1->data : (NCSTRING)\"\")" % [a.rdLoc],
cgCall(p, "nimToCStringConv", arg),
a.storage)
proc convCStrToStr(p: BProc, n: PNode, d: var TLoc) =
@@ -2789,13 +2806,19 @@ proc genMove(p: BProc; n: PNode; d: var TLoc) =
var src: TLoc = initLocExpr(p, n[2])
let destVal = rdLoc(a)
let srcVal = rdLoc(src)
p.s(cpsStmts).addSingleIfStmt(
cOp(NotEqual,
dotField(destVal, "p"),
dotField(srcVal, "p"))):
if p.config.isDefined("nimsso") and
n[1].typ.skipTypes(abstractVar).kind == tyString:
# SmallString: destroy dst then struct-copy src; no .p field aliasing needed
genStmts(p, n[3])
p.s(cpsStmts).addFieldAssignment(destVal, "len", dotField(srcVal, "len"))
p.s(cpsStmts).addFieldAssignment(destVal, "p", dotField(srcVal, "p"))
genAssignment(p, a, src, {})
else:
p.s(cpsStmts).addSingleIfStmt(
cOp(NotEqual,
dotField(destVal, "p"),
dotField(srcVal, "p"))):
genStmts(p, n[3])
p.s(cpsStmts).addFieldAssignment(destVal, "len", dotField(srcVal, "len"))
p.s(cpsStmts).addFieldAssignment(destVal, "p", dotField(srcVal, "p"))
else:
if d.k == locNone: d = getTemp(p, n.typ)
if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}:
@@ -2832,15 +2855,19 @@ proc genDestroy(p: BProc; n: PNode) =
case t.kind
of tyString:
var a: TLoc = initLocExpr(p, arg)
let ra = rdLoc(a)
let rp = dotField(ra, "p")
p.s(cpsStmts).addSingleIfStmt(
cOp(And, rp,
cOp(Not, cOp(BitAnd, NimInt,
derefField(rp, "cap"),
NimStrlitFlag)))):
let fn = if optThreads in p.config.globalOptions: "deallocShared" else: "dealloc"
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, fn), rp)
if p.config.isDefined("nimsso"):
# SmallString: delegate to nimDestroyStrV1 (rc-based, handles static strings)
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, "nimDestroyStrV1"), rdLoc(a))
else:
let ra = rdLoc(a)
let rp = dotField(ra, "p")
p.s(cpsStmts).addSingleIfStmt(
cOp(And, rp,
cOp(Not, cOp(BitAnd, NimInt,
derefField(rp, "cap"),
NimStrlitFlag)))):
let fn = if optThreads in p.config.globalOptions: "deallocShared" else: "dealloc"
p.s(cpsStmts).addCallStmt(cgsymValue(p.module, fn), rp)
of tySequence:
var a: TLoc = initLocExpr(p, arg)
let ra = rdLoc(a)
@@ -4200,7 +4227,10 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; resul
genConstObjConstr(p, n, isConst, result)
of tyString, tyCstring:
if optSeqDestructors in p.config.globalOptions and n.kind != nkNilLit and ty == tyString:
genStringLiteralV2Const(p.module, n, isConst, result)
if p.config.isDefined("nimsso"):
genStringLiteralV3Const(p.module, n, isConst, result)
else:
genStringLiteralV2Const(p.module, n, isConst, result)
else:
var d: TLoc = initLocExpr(p, n)
result.add rdLoc(d)

View File

@@ -22,7 +22,11 @@ template detectVersion(field, corename) =
result = 1
proc detectStrVersion(m: BModule): int =
detectVersion(strVersion, "nimStrVersion")
if m.g.config.isDefined("nimsso") and
m.g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc, gcHooks}:
result = 3
else:
detectVersion(strVersion, "nimStrVersion")
proc detectSeqVersion(m: BModule): int =
detectVersion(seqVersion, "nimSeqVersion")
@@ -128,6 +132,155 @@ proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Bu
result.addField(strInit, name = "p"):
result.add(cCast(ptrType("NimStrPayload"), cAddr(pureLit)))
proc ssoCharLit(ch: char): string =
## Return a C char literal for ch, with proper escaping.
const hexDigits = "0123456789abcdef"
result = "'"
case ch
of '\'': result.add("\\'")
of '\\': result.add("\\\\")
of '\0': result.add("\\0")
of '\n': result.add("\\n")
of '\r': result.add("\\r")
of '\t': result.add("\\t")
elif ch.ord < 32 or ch.ord == 127:
result.add("\\x")
result.add(hexDigits[ch.ord shr 4])
result.add(hexDigits[ch.ord and 0xf])
else:
result.add(ch)
result.add('\'')
proc ssoPayloadLit(src: string; maxLen: int): string =
const AlwaysAvail = 7
result = "{"
for i in 0..<AlwaysAvail:
if i > 0: result.add(',')
let ch = if i < maxLen: src[i] else: '\0'
result.add(ssoCharLit(ch))
result.add('}')
proc genStringLiteralV3Const(m: BModule; n: PNode; isConst: bool; result: var Builder) =
# Inline SmallString struct initializer for use inside const aggregate types.
# Short strings (<=7 chars) embed all chars directly. Long strings reference
# a static LongString block emitted separately into cfsStrData.
const AlwaysAvail = 7
let s = n.strVal
cgsym(m, "SmallString")
cgsym(m, "LongString")
var si: StructInitializer
result.addStructInitializer(si, kind = siOrderedStruct):
if s.len <= AlwaysAvail:
result.addField(si, name = "slen"):
result.addIntValue(s.len)
result.addField(si, name = "payload"):
result.add(ssoPayloadLit(s, s.len))
result.addField(si, name = "more"):
result.add(NimNil)
else:
# Emit the LongString block into cfsStrData and reference it inline.
let dataName = getTempName(m)
var res = newBuilder("")
res.addVarWithTypeAndInitializer(
if isConst: AlwaysConst else: Global,
name = dataName):
res.addSimpleStruct(m, name = "", baseType = ""):
res.addField(name = "rc", typ = NimInt)
res.addField(name = "fullLen", typ = NimInt)
res.addField(name = "capImpl", typ = NimInt)
res.addArrayField(name = "data", elementType = NimChar, len = s.len + 1)
do:
var di: StructInitializer
res.addStructInitializer(di, kind = siOrderedStruct):
res.addField(di, name = "rc"):
res.addIntValue(1)
res.addField(di, name = "fullLen"):
res.addIntValue(s.len)
res.addField(di, name = "capImpl"):
res.addIntValue(0) # static, never freed
res.addField(di, name = "data"):
res.add(makeCString(s))
m.s[cfsStrData].add(extract(res))
result.addField(si, name = "slen"):
result.addIntValue(255)
result.addField(si, name = "payload"):
result.add(ssoPayloadLit(s, AlwaysAvail))
result.addField(si, name = "more"):
result.add(cCast(ptrType("LongString"), cAddr(dataName)))
# ------ Version 3: SmallString (SSO) strings --------------------------------
proc genStringLiteralV3(m: BModule; n: PNode; isConst: bool; result: var Builder) =
# SmallString literal. Always generate a fresh SmallString variable (like v2
# always generates a fresh outer NimStringV2). For long strings, cache the
# LongString payload to avoid duplicates within a module.
const AlwaysAvail = 7 # must match strs_v3.nim
let s = n.strVal
let tmp = getTempName(m)
result.add tmp
cgsym(m, "SmallString")
cgsym(m, "LongString")
var res = newBuilder("")
if s.len <= AlwaysAvail:
# Short: all chars fit in payload, more = NULL.
res.addVarWithInitializer(
if isConst: AlwaysConst else: Global,
name = tmp, typ = "SmallString"):
var si: StructInitializer
res.addStructInitializer(si, kind = siOrderedStruct):
res.addField(si, name = "slen"):
res.addIntValue(s.len)
res.addField(si, name = "payload"):
res.add(ssoPayloadLit(s, s.len))
res.addField(si, name = "more"):
res.add(NimNil)
else:
# Long: cache the LongString block to emit it only once per module per string.
# Always generate a fresh SmallString pointing at the (possibly cached) block.
let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
var dataName: string
if id == m.labels:
dataName = getTempName(m)
res.addVarWithTypeAndInitializer(
if isConst: AlwaysConst else: Global,
name = dataName):
res.addSimpleStruct(m, name = "", baseType = ""):
res.addField(name = "rc", typ = NimInt)
res.addField(name = "fullLen", typ = NimInt)
res.addField(name = "capImpl", typ = NimInt)
res.addArrayField(name = "data", elementType = NimChar, len = s.len + 1)
do:
var di: StructInitializer
res.addStructInitializer(di, kind = siOrderedStruct):
res.addField(di, name = "rc"):
res.addIntValue(1)
res.addField(di, name = "fullLen"):
res.addIntValue(s.len)
res.addField(di, name = "capImpl"):
res.addIntValue(0) # bit 0 = 0: static, never freed
res.addField(di, name = "data"):
res.add(makeCString(s))
else:
dataName = m.tmpBase & $id
# PayloadSize = AlwaysAvail + sizeof(pointer) - 1; sentinel slen = PayloadSize+1
# We just use a large value (255) that is guaranteed > PayloadSize on all platforms.
res.addVarWithInitializer(
if isConst: AlwaysConst else: Global,
name = tmp, typ = "SmallString"):
var si: StructInitializer
res.addStructInitializer(si, kind = siOrderedStruct):
res.addField(si, name = "slen"):
res.addIntValue(255) # > PayloadSize on all platforms => long sentinel
res.addField(si, name = "payload"):
res.add(ssoPayloadLit(s, AlwaysAvail))
res.addField(si, name = "more"):
res.add(cCast(ptrType("LongString"), cAddr(dataName)))
m.s[cfsStrData].add(extract(res))
# ------ Version selector ---------------------------------------------------
proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
@@ -138,6 +291,8 @@ proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
let tmp = getTempName(m)
genStringLiteralDataOnlyV2(m, s, tmp, isConst)
result.add tmp
of 3:
localError(m.config, info, "genStringLiteralDataOnly not supported for SmallString (nimsso)")
else:
localError(m.config, info, "cannot determine how to produce code for string literal")
@@ -148,5 +303,6 @@ proc genStringLiteral(m: BModule; n: PNode; result: var Builder) =
case detectStrVersion(m)
of 0, 1: genStringLiteralV1(m, n, result)
of 2: genStringLiteralV2(m, n, isConst = true, result)
of 3: genStringLiteralV3(m, n, isConst = true, result)
else:
localError(m.config, n.info, "cannot determine how to produce code for string literal")

View File

@@ -339,6 +339,10 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope =
cgsym(m, "NimStrPayload")
cgsym(m, "NimStringV2")
result = typeNameOrLiteral(m, typ, "NimStringV2")
of 3:
cgsym(m, "LongString")
cgsym(m, "SmallString")
result = typeNameOrLiteral(m, typ, "SmallString")
else:
cgsym(m, "NimStringDesc")
result = typeNameOrLiteral(m, typ, "NimStringDesc*")

View File

@@ -389,7 +389,11 @@ proc lenField(p: BProc, val: Rope): Rope {.inline.} =
proc lenExpr(p: BProc; a: TLoc): Rope =
if optSeqDestructors in p.config.globalOptions:
result = dotField(rdLoc(a), "len")
if p.config.isDefined("nimsso") and a.t != nil and
a.t.skipTypes(abstractInst).kind == tyString:
result = cCall(cgsymValue(p.module, "nimStrLen"), rdLoc(a))
else:
result = dotField(rdLoc(a), "len")
else:
let ra = rdLoc(a)
result = cIfExpr(ra, lenField(p, ra), cIntValue(0))
@@ -530,7 +534,15 @@ proc resetLoc(p: BProc, loc: var TLoc) =
let atyp = skipTypes(loc.t, abstractInst)
let rl = rdLoc(loc)
if atyp.kind in {tyVar, tyLent}:
if typ.kind == tyString and p.config.isDefined("nimsso"):
# SmallString zero state: slen=0 suffices (slen<=AlwaysAvail => inline, no heap)
if atyp.kind in {tyVar, tyLent}:
p.s(cpsStmts).addAssignment(derefField(rl, "slen"), cIntValue(0))
p.s(cpsStmts).addAssignment(derefField(rl, "more"), NimNil)
else:
p.s(cpsStmts).addAssignment(dotField(rl, "slen"), cIntValue(0))
p.s(cpsStmts).addAssignment(dotField(rl, "more"), NimNil)
elif atyp.kind in {tyVar, tyLent}:
p.s(cpsStmts).addAssignment(derefField(rl, "len"), cIntValue(0))
p.s(cpsStmts).addAssignment(derefField(rl, "p"), NimNil)
else:
@@ -580,8 +592,13 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) =
let typ = loc.t
if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}:
let rl = rdLoc(loc)
p.s(cpsStmts).addFieldAssignment(rl, "len", cIntValue(0))
p.s(cpsStmts).addFieldAssignment(rl, "p", NimNil)
if skipTypes(typ, abstractInst + {tyStatic}).kind == tyString and p.config.isDefined("nimsso"):
# SmallString zero state: slen=0 suffices
p.s(cpsStmts).addFieldAssignment(rl, "slen", cIntValue(0))
p.s(cpsStmts).addFieldAssignment(rl, "more", NimNil)
else:
p.s(cpsStmts).addFieldAssignment(rl, "len", cIntValue(0))
p.s(cpsStmts).addFieldAssignment(rl, "p", NimNil)
elif not isComplexValueType(typ):
if containsGarbageCollectedRef(loc.t):
var nilLoc: TLoc = initLoc(locTemp, loc.lode, OnStack)

View File

@@ -701,11 +701,18 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
of attachedAsgn, attachedDeepCopy, attachedDup:
body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c, x), y)
of attachedSink:
let moveCall = genBuiltin(c, mMove, "move", x)
moveCall.add y
doAssert t.destructor != nil
moveCall.add destructorCall(c, t.destructor, x)
body.add moveCall
if c.g.config.isDefined("nimsso"):
# SmallString: destroy old dst, then bit-copy src (no rc increment — this is a move).
# No .p aliasing check needed; rc-based destroy handles COW sharing correctly.
doAssert t.destructor != nil
body.add destructorCall(c, t.destructor, x)
body.add newAsgnStmt(x, y)
else:
let moveCall = genBuiltin(c, mMove, "move", x)
moveCall.add y
doAssert t.destructor != nil
moveCall.add destructorCall(c, t.destructor, x)
body.add moveCall
of attachedDestructor:
body.add genBuiltin(c, mDestroy, "destroy", x)
of attachedTrace:

View File

@@ -1641,7 +1641,7 @@ when defined(windows):
const ERROR_BAD_EXE_FORMAT = 193
when notJSnotNims:
when defined(nimSeqsV2):
when defined(nimSeqsV2) and not defined(nimsso):
proc nimToCStringConv(s: NimStringV2): cstring {.compilerproc, nonReloadable, inline.}
when hostOS != "standalone" and hostOS != "any":
@@ -1689,7 +1689,10 @@ when not defined(nimIcIntegrityChecks):
export exceptions
when notJSnotNims and defined(nimSeqsV2):
include "system/strs_v2"
when defined(nimsso):
include "system/strs_v3"
else:
include "system/strs_v2"
include "system/seqs_v2"
when not defined(js):

View File

@@ -62,9 +62,14 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) =
case mt.kind
of tyString:
when defined(nimSeqsV2):
var x = cast[ptr NimStringV2](dest)
var s2 = cast[ptr NimStringV2](s)[]
nimAsgnStrV2(x[], s2)
when defined(nimsso):
var x = cast[ptr SmallString](dest)
var s2 = cast[ptr SmallString](s)[]
nimAsgnStrV2(x[], s2)
else:
var x = cast[ptr NimStringV2](dest)
var s2 = cast[ptr NimStringV2](s)[]
nimAsgnStrV2(x[], s2)
else:
var x = cast[PPointer](dest)
var s2 = cast[PPointer](s)[]
@@ -245,8 +250,11 @@ proc genericReset(dest: pointer, mt: PNimType) =
unsureAsgnRef(cast[PPointer](dest), nil)
of tyString:
when defined(nimSeqsV2):
var s = cast[ptr NimStringV2](dest)
frees(s[])
when defined(nimsso):
nimDestroyStrV1(cast[ptr SmallString](dest)[])
else:
var s = cast[ptr NimStringV2](dest)
frees(s[])
zeroMem(dest, mt.size)
else:
unsureAsgnRef(cast[PPointer](dest), nil)

View File

@@ -92,9 +92,14 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) =
case mt.kind
of tyString:
when defined(nimSeqsV2):
var x = cast[ptr NimStringV2](dest)
var s2 = cast[ptr NimStringV2](s)[]
nimAsgnStrV2(x[], s2)
when defined(nimsso):
var x = cast[ptr SmallString](dest)
var s2 = cast[ptr SmallString](s)[]
nimAsgnStrV2(x[], s2)
else:
var x = cast[ptr NimStringV2](dest)
var s2 = cast[ptr NimStringV2](s)[]
nimAsgnStrV2(x[], s2)
else:
var x = cast[PPointer](dest)
var s2 = cast[PPointer](s)[]

495
lib/system/strs_v3.nim Normal file
View File

@@ -0,0 +1,495 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2026 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Small String Optimization (SSO) implementation used by Nim's core.
const
AlwaysAvail = 7
PayloadSize = AlwaysAvail + sizeof(pointer) - 1 # -1 reserves the last byte for '\0'
proc atomicAddFetch(p: var int; v: int): int {.importc: "__sync_add_and_fetch", nodecl.}
proc atomicSubFetch(p: var int; v: int): int {.importc: "__sync_sub_and_fetch", nodecl.}
type
LongString {.core.} = object
rc: int # atomic reference count; 1 = unique owner
fullLen: int
capImpl: int # bit 0: heap-allocated; upper bits: capacity (cap = capImpl shr 1)
data: UncheckedArray[char]
SmallString {.core.} = object
slen: byte # when > PayloadSize, `more` is valid ptr
payload: array[AlwaysAvail, char]
more: ptr LongString # when long: pointer; when small (len 8..15): bytes 7..14 stored here
proc resize(old: int): int {.inline.} =
## Capacity growth factor shared with seqs_v2.nim.
if old <= 0: result = 4
elif old <= high(int16): result = old * 2
else: result = old div 2 + old
proc `=destroy`*(s: var SmallString) =
if int(s.slen) > PayloadSize and (s.more.capImpl and 1) == 1:
if atomicSubFetch(s.more.rc, 1) == 0:
dealloc(s.more)
proc `=wasMoved`*(s: var SmallString) {.inline.} =
s.slen = 0
proc `=sink`*(dst: var SmallString; src: SmallString) =
`=destroy`(dst)
copyMem(addr dst, unsafeAddr src, sizeof(SmallString))
proc `=copy`*(dst: var SmallString; src: SmallString) =
if int(src.slen) <= PayloadSize:
`=destroy`(dst) # dst may have been a long string
copyMem(addr dst, unsafeAddr src, sizeof(SmallString))
else:
if addr(dst) == unsafeAddr(src): return
`=destroy`(dst)
# COW: share the block, bump refcount — no allocation needed
if (src.more.capImpl and 1) == 1:
discard atomicAddFetch(src.more.rc, 1)
copyMem(addr dst, unsafeAddr src, sizeof(SmallString))
proc `=dup`*(src: SmallString): SmallString =
copyMem(addr result, unsafeAddr src, sizeof(SmallString))
if int(src.slen) > PayloadSize and (src.more.capImpl and 1) == 1:
discard atomicAddFetch(src.more.rc, 1)
proc ensureUniqueLong(s: var SmallString; oldLen, newLen: int) =
# Ensure s.more is a unique (rc=1) heap block with capacity >= newLen, preserving existing data.
# s must already be a long string on entry.
let heapAlloc = (s.more.capImpl and 1) == 1
let unique = heapAlloc and s.more.rc == 1
let cap = s.more.capImpl shr 1
if unique and newLen <= cap:
s.more.fullLen = newLen
else:
let newCap = max(newLen, oldLen * 2)
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + newCap + 1))
p.rc = 1
p.fullLen = newLen
p.capImpl = (newCap shl 1) or 1
let old = s.more
copyMem(addr p.data[0], addr old.data[0], oldLen + 1) # +1 preserves the '\0'
if heapAlloc and atomicSubFetch(old.rc, 1) == 0:
dealloc(old)
s.more = p
proc len*(s: SmallString): int {.inline.} =
result = int s.slen
if result > PayloadSize:
result = s.more.fullLen
template guts(s: SmallString): (int, ptr UncheckedArray[char]) =
let slen = int s.slen
if slen > PayloadSize:
(s.more.fullLen, cast[ptr UncheckedArray[char]](addr s.more.data[0]))
else:
(slen, cast[ptr UncheckedArray[char]](addr s.payload[0]))
proc `[]`*(s: SmallString; i: int): char {.inline.} =
let slen = int s.slen
if slen <= PayloadSize:
# unchecked: when i >= 7 we store into the `more` overlay
result = (cast[ptr UncheckedArray[char]](addr s.payload[0]))[i]
elif i < AlwaysAvail:
result = s.payload[i]
else:
result = s.more.data[i]
proc `[]=`*(s: var SmallString; i: int; c: char) =
let slen = int s.slen
if slen <= PayloadSize:
# unchecked: when i >= 7 we store into the `more` overlay
(cast[ptr UncheckedArray[char]](addr s.payload[0]))[i] = c
else:
let l = s.more.fullLen
ensureUniqueLong(s, l, l) # COW if shared; length unchanged
s.more.data[i] = c
if i < AlwaysAvail:
s.payload[i] = c
proc cmp*(a, b: SmallString): int =
# Use slen directly for prefix length: for short/medium it is the real length,
# for long it is the sentinel (> AlwaysAvail), so min(..., AlwaysAvail) still gives 7.
# This avoids dereferencing `more` before the prefix comparison.
let pfxLen = min(min(int a.slen, int b.slen), AlwaysAvail)
result = cmpMem(unsafeAddr a.payload[0], unsafeAddr b.payload[0], pfxLen)
if result != 0: return
# Prefix matched — now fetch actual lengths (dereferences `more` only if long)
let la = if int(a.slen) > PayloadSize: a.more.fullLen else: int(a.slen)
let lb = if int(b.slen) > PayloadSize: b.more.fullLen else: int(b.slen)
let minLen = min(la, lb)
if minLen <= AlwaysAvail:
result = la - lb
return
let (_, pa) = a.guts
let (_, pb) = b.guts
result = cmpMem(addr pa[AlwaysAvail], addr pb[AlwaysAvail], minLen - AlwaysAvail)
if result == 0:
result = la - lb
proc `==`*(a, b: SmallString): bool =
if a.slen != b.slen: return false
# slen equal: for short/medium this means equal lengths; for long (both sentinel) we still need fullLen.
let slen = int(a.slen)
let pfxLen = min(slen, AlwaysAvail)
if cmpMem(unsafeAddr a.payload[0], unsafeAddr b.payload[0], pfxLen) != 0: return false
if slen <= AlwaysAvail: return true
if slen <= PayloadSize:
# medium: guts gives the UncheckedArray without a heap dereference
let (la, pa) = a.guts
let (_, pb) = b.guts
return cmpMem(addr pa[pfxLen], addr pb[pfxLen], la - pfxLen) == 0
# long: fetch actual lengths only after prefix matched
let la = a.more.fullLen
if la != b.more.fullLen: return false
cmpMem(addr a.more.data[pfxLen], addr b.more.data[pfxLen], la - pfxLen) == 0
proc `<=`*(a, b: SmallString): bool {.inline.} = cmp(a, b) <= 0
proc continuesWith*(s, sub: SmallString; start: int): bool =
if start < 0: return false
let subslen = int(sub.slen)
if subslen == 0: return true
# Compare inline prefix first — no `more` dereference yet.
# For long sub, subslen is the sentinel (> AlwaysAvail), so pfxLen is capped correctly.
let pfxLen = min(subslen, max(0, AlwaysAvail - start))
if pfxLen > 0:
if cmpMem(unsafeAddr s.payload[start], unsafeAddr sub.payload[0], pfxLen) != 0:
return false
# Prefix matched (or start >= AlwaysAvail); now fetch actual lengths
let subLen = if subslen > PayloadSize: sub.more.fullLen else: subslen
let sLen = if int(s.slen) > PayloadSize: s.more.fullLen else: int(s.slen)
if start + subLen > sLen: return false
if pfxLen == subLen: return true # sub fully compared within the prefix
let (_, sp) = s.guts
let (_, subp) = sub.guts
cmpMem(addr sp[start + pfxLen], addr subp[pfxLen], subLen - pfxLen) == 0
proc startsWith*(s, sub: SmallString): bool {.inline.} = continuesWith(s, sub, 0)
proc endsWith*(s, sub: SmallString): bool {.inline.} = continuesWith(s, sub, s.len - sub.len)
proc add*(s: var SmallString; c: char) =
let slen = int(s.slen)
if slen <= PayloadSize:
let newLen = slen + 1
if newLen <= PayloadSize:
let inl = cast[ptr UncheckedArray[char]](addr s.payload[0])
inl[slen] = c
inl[newLen] = '\0'
s.slen = byte(newLen)
else:
# transition from medium (slen == PayloadSize) to long
let cap = newLen * 2
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + cap + 1))
p.rc = 1
p.fullLen = newLen
p.capImpl = (cap shl 1) or 1
copyMem(addr p.data[0], cast[ptr UncheckedArray[char]](addr s.payload[0]), slen)
p.data[slen] = c
p.data[newLen] = '\0'
# payload[0..AlwaysAvail-1] already correct; slen >= AlwaysAvail so no update needed
s.more = p
s.slen = byte(PayloadSize + 1)
else:
let l = s.more.fullLen # fetch fullLen only in the long path
ensureUniqueLong(s, l, l + 1)
s.more.data[l] = c
s.more.data[l + 1] = '\0'
# l >= PayloadSize > AlwaysAvail, so prefix is unaffected
proc add*(s: var SmallString; t: SmallString) =
let slen = int(s.slen)
let (tl, tp) = t.guts # fetch t's guts before any mutation (aliasing safety)
if tl == 0: return
if slen <= PayloadSize:
let sl = slen # for short/medium, slen IS the actual length
let newLen = sl + tl
if newLen <= PayloadSize:
let inl = cast[ptr UncheckedArray[char]](addr s.payload[0])
copyMem(addr inl[sl], tp, tl)
inl[newLen] = '\0'
s.slen = byte(newLen)
else:
# transition to long
let cap = newLen * 2
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + cap + 1))
p.rc = 1
p.fullLen = newLen
p.capImpl = (cap shl 1) or 1
copyMem(addr p.data[0], cast[ptr UncheckedArray[char]](addr s.payload[0]), sl)
copyMem(addr p.data[sl], tp, tl)
p.data[newLen] = '\0'
# update prefix bytes that come from t (only when sl < AlwaysAvail)
if sl < AlwaysAvail:
copyMem(addr s.payload[sl], tp, min(AlwaysAvail - sl, tl))
s.more = p
s.slen = byte(PayloadSize + 1)
else:
let sl = s.more.fullLen # fetch fullLen only in the long path
let newLen = sl + tl
# tp was read before ensureUniqueLong: if t.more == s.more, rc decrements but won't hit 0
ensureUniqueLong(s, sl, newLen)
copyMem(addr s.more.data[sl], tp, tl)
s.more.data[newLen] = '\0'
# sl >= PayloadSize > AlwaysAvail, so prefix is unaffected
proc `&`*(a, b: SmallString): SmallString =
result = a
result.add(b)
proc toSmallString*(s: openArray[char]): SmallString =
let l = s.len
if l == 0: return
if l <= PayloadSize:
result.slen = byte(l)
let inl = cast[ptr UncheckedArray[char]](addr result.payload[0])
copyMem(inl, unsafeAddr s[0], l)
inl[l] = '\0'
else:
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + l + 1))
p.rc = 1
p.fullLen = l
p.capImpl = (l shl 1) or 1
copyMem(addr p.data[0], unsafeAddr s[0], l)
p.data[l] = '\0'
copyMem(addr result.payload[0], unsafeAddr s[0], AlwaysAvail)
result.slen = byte(PayloadSize + 1)
result.more = p
{.push overflowChecks: off, rangeChecks: off.}
proc prepareAddLong(s: var SmallString; newLen: int) =
# Reserve capacity for newLen in the long-string block without changing logical length.
let heapAlloc = (s.more.capImpl and 1) == 1
let cap = s.more.capImpl shr 1
if heapAlloc and s.more.rc == 1 and newLen <= cap:
discard # already unique with sufficient capacity
else:
let oldLen = s.more.fullLen
let newCap = max(newLen, oldLen * 2)
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + newCap + 1))
p.rc = 1
p.fullLen = oldLen # logical length unchanged — caller sets it after writing data
p.capImpl = (newCap shl 1) or 1
let old = s.more
copyMem(addr p.data[0], addr old.data[0], oldLen + 1)
if heapAlloc and atomicSubFetch(old.rc, 1) == 0:
dealloc(old)
s.more = p
proc prepareAdd*(s: var SmallString; addLen: int) {.compilerRtl.} =
## Ensure s has room for addLen more characters without changing its length.
let slen = int(s.slen)
let curLen = if slen > PayloadSize: s.more.fullLen else: slen
let newLen = curLen + addLen
if slen <= PayloadSize:
if newLen > PayloadSize:
# transition to long: allocate, copy existing data
let newCap = newLen * 2
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + newCap + 1))
p.rc = 1
p.fullLen = curLen
p.capImpl = (newCap shl 1) or 1
let inl = cast[ptr UncheckedArray[char]](addr s.payload[0])
copyMem(addr p.data[0], inl, curLen + 1)
s.more = p
s.slen = byte(PayloadSize + 1)
# else: short/medium — inline capacity always sufficient (struct is fixed size)
else:
prepareAddLong(s, newLen)
proc nimAddCharV1*(s: var SmallString; c: char) {.compilerRtl, inline.} =
prepareAdd(s, 1)
s.add(c)
proc toNimStr*(str: cstring; len: int): SmallString {.compilerproc.} =
if len <= 0: return
if len <= PayloadSize:
result.slen = byte(len)
let inl = cast[ptr UncheckedArray[char]](addr result.payload[0])
copyMem(inl, str, len)
inl[len] = '\0'
else:
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + len + 1))
p.rc = 1
p.fullLen = len
p.capImpl = (len shl 1) or 1
copyMem(addr p.data[0], str, len)
p.data[len] = '\0'
copyMem(addr result.payload[0], str, AlwaysAvail)
result.slen = byte(PayloadSize + 1)
result.more = p
proc cstrToNimstr*(str: cstring): SmallString {.compilerRtl.} =
if str == nil: return
toNimStr(str, str.len)
proc nimToCStringConv*(s: var SmallString): cstring {.compilerproc, nonReloadable, inline.} =
## Returns a null-terminated C string pointer into s's data.
## Takes by var (pointer) so addr s.payload[0] is always into the caller's SmallString.
if int(s.slen) > PayloadSize:
cast[cstring](addr s.more.data[0])
else:
cast[cstring](addr s.payload[0])
proc appendString*(dest: var SmallString; src: SmallString) {.compilerproc, inline.} =
dest.add(src)
proc appendChar*(dest: var SmallString; c: char) {.compilerproc, inline.} =
dest.add(c)
proc rawNewString*(space: int): SmallString {.compilerproc.} =
## Returns an empty SmallString with capacity reserved for `space` chars (newStringOfCap).
if space <= 0: return
if space <= PayloadSize:
discard # inline capacity is always available; nothing to pre-allocate
else:
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + space + 1))
p.rc = 1
p.fullLen = 0
p.capImpl = (space shl 1) or 1
p.data[0] = '\0'
result.more = p
result.slen = byte(PayloadSize + 1)
proc mnewString*(len: int): SmallString {.compilerproc.} =
## Returns a SmallString of `len` zero characters (newString).
if len <= 0: return
if len <= PayloadSize:
result.slen = byte(len)
# payload is zero-initialized by default (result is zero)
cast[ptr UncheckedArray[char]](addr result.payload[0])[len] = '\0'
else:
let p = cast[ptr LongString](alloc0(sizeof(int) * 3 + len + 1))
p.rc = 1
p.fullLen = len
p.capImpl = (len shl 1) or 1
# data is zeroed by alloc0; data[len] is '\0' too
result.more = p
result.slen = byte(PayloadSize + 1)
proc setLengthStrV2*(s: var SmallString; newLen: int) {.compilerRtl.} =
## Sets the length of s to newLen, zeroing new bytes on growth.
let slen = int(s.slen)
let curLen = if slen > PayloadSize: s.more.fullLen else: slen
if newLen == curLen: return
if newLen <= 0:
if slen > PayloadSize:
if (s.more.capImpl and 1) == 1 and s.more.rc == 1:
s.more.fullLen = 0
s.more.data[0] = '\0'
else:
# shared block: detach and go back to empty inline
`=destroy`(s)
s.slen = 0
else:
s.slen = 0
s.payload[0] = '\0'
return
if slen <= PayloadSize:
if newLen <= PayloadSize:
if newLen > curLen:
let inl = cast[ptr UncheckedArray[char]](addr s.payload[0])
zeroMem(addr inl[curLen], newLen - curLen)
inl[newLen] = '\0'
else:
cast[ptr UncheckedArray[char]](addr s.payload[0])[newLen] = '\0'
s.slen = byte(newLen)
else:
# grow into long
let newCap = newLen * 2
let p = cast[ptr LongString](alloc0(sizeof(int) * 3 + newCap + 1))
p.rc = 1
p.fullLen = newLen
p.capImpl = (newCap shl 1) or 1
copyMem(addr p.data[0], cast[ptr UncheckedArray[char]](addr s.payload[0]), curLen)
# bytes [curLen..newLen] zeroed by alloc0; p.data[newLen] = '\0' by alloc0
s.more = p
s.slen = byte(PayloadSize + 1)
else:
# currently long
if newLen <= PayloadSize:
# shrink back to inline
let old = s.more
let heapAlloc = (old.capImpl and 1) == 1
let inl = cast[ptr UncheckedArray[char]](addr s.payload[0])
copyMem(inl, addr old.data[0], newLen)
inl[newLen] = '\0'
if heapAlloc and atomicSubFetch(old.rc, 1) == 0:
dealloc(old)
s.slen = byte(newLen)
else:
ensureUniqueLong(s, curLen, newLen)
if newLen > curLen:
zeroMem(addr s.more.data[curLen], newLen - curLen)
s.more.data[newLen] = '\0'
s.more.fullLen = newLen
proc nimAsgnStrV2*(a: var SmallString; b: SmallString) {.compilerRtl.} =
`=copy`(a, b)
proc nimPrepareStrMutationImpl(s: var SmallString) =
# Called when s holds a static (non-heap) LongString block. COW: allocate a fresh copy.
let old = s.more
let oldLen = old.fullLen
let p = cast[ptr LongString](alloc(sizeof(int) * 3 + oldLen + 1))
p.rc = 1
p.fullLen = oldLen
p.capImpl = (oldLen shl 1) or 1
copyMem(addr p.data[0], addr old.data[0], oldLen + 1)
s.more = p
proc nimPrepareStrMutationV2*(s: var SmallString) {.compilerRtl, inline.} =
if int(s.slen) > PayloadSize and (s.more.capImpl and 1) == 0:
nimPrepareStrMutationImpl(s)
proc prepareMutation*(s: var string) {.inline.} =
{.cast(noSideEffect).}:
nimPrepareStrMutationV2(cast[ptr SmallString](addr s)[])
proc nimAddStrV1*(s: var SmallString; src: SmallString) {.compilerRtl, inline.} =
s.add(src)
proc nimDestroyStrV1*(s: SmallString) {.compilerRtl, inline.} =
if int(s.slen) > PayloadSize and (s.more.capImpl and 1) == 1:
if atomicSubFetch(s.more.rc, 1) == 0:
dealloc(s.more)
proc nimStrAtLe*(s: SmallString; idx: int; ch: char): bool {.compilerRtl, inline.} =
let l = s.len
result = idx < l and s[idx] <= ch
func capacity*(self: SmallString): int {.inline.} =
## Returns the current capacity of the string.
let slen = int(self.slen)
if slen > PayloadSize:
self.more.capImpl shr 1
else:
PayloadSize
proc nimStrLen*(s: SmallString): int {.compilerproc, inline.} =
## Returns the length of s. Called by the codegen for `mLen` on strings with -d:nimsso.
s.len
proc nimStrData*(s: var SmallString): ptr UncheckedArray[char] {.compilerproc, inline.} =
## Returns a pointer to the char data of s. Called by codegen for subscript and slice with -d:nimsso.
let slen = int(s.slen)
if slen > PayloadSize: cast[ptr UncheckedArray[char]](addr s.more.data[0])
else: cast[ptr UncheckedArray[char]](addr s.payload[0])
proc eqStrings*(a, b: SmallString): bool {.compilerproc, inline.} = a == b
proc cmpStrings*(a, b: SmallString): int {.compilerproc, inline.} = cmp(a, b)
{.pop.}