diff --git a/changelog.md b/changelog.md index 33f36deec2..cf2d4cb170 100644 --- a/changelog.md +++ b/changelog.md @@ -23,6 +23,8 @@ errors. - With `-d:nimPreviewCStringComparisons`, comparsions (`<`, `>`, `<=`, `>=`) between cstrings switch from reference semantics to value semantics like `==` and `!=`. +- `std/parsesql` has been moved to a nimble package, use `nimble` or `atlas` to install it. + ## Standard library additions and changes [//]: # "Additions:" @@ -33,6 +35,20 @@ errors. - `strutils.multiReplace` overload for character set replacements in a single pass. Useful for string sanitation. Follows existing multiReplace semantics. +- `std/files` adds: + - Exports `CopyFlag` enum and `FilePermission` type for fine-grained control of file operations + - New file operation procs with `Path` support: + - `getFilePermissions`, `setFilePermissions` for managing permissions + - `tryRemoveFile` for file deletion + - `copyFile` with configurable buffer size and symlink handling + - `copyFileWithPermissions` to preserve file attributes + - `copyFileToDir` for copying files into directories + +- `std/dirs` adds: + - New directory operation procs with `Path` support: + - `copyDir` with special file handling options + - `copyDirWithPermissions` to recursively preserve attributes + - `system.setLenUninit` now supports refc, JS and VM backends. [//]: # "Changes:" diff --git a/compiler/ast.nim b/compiler/ast.nim index d1d2d127a0..d80589c087 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -819,6 +819,7 @@ type # nodes are compared by structure! counter*: int data*: TNodePairSeq + ignoreTypes*: bool TObjectSeq* = seq[RootRef] TObjectSet* = object @@ -1641,8 +1642,8 @@ proc initObjectSet*(): TObjectSet = result = TObjectSet(counter: 0) newSeq(result.data, StartSize) -proc initNodeTable*(): TNodeTable = - result = TNodeTable(counter: 0) +proc initNodeTable*(ignoreTypes=false): TNodeTable = + result = TNodeTable(counter: 0, ignoreTypes: ignoreTypes) newSeq(result.data, StartSize) proc skipTypes*(t: PType, kinds: TTypeKinds; maxIters: int): PType = diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 4ca34b9d74..d3e215ea56 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -981,7 +981,11 @@ proc genAddr(p: BProc, e: PNode, d: var TLoc) = var a: TLoc = initLocExpr(p, e[0]) if e[0].kind in {nkHiddenStdConv, nkHiddenSubConv, nkConv} and not ignoreConv(e[0]): # addr (conv x) introduces a temp because `conv x` is not a rvalue - putIntoDest(p, d, e, addrLoc(p.config, expressionsNeedsTmp(p, a)), a.storage) + # transform addr ( conv ( x ) ) -> conv ( addr ( x ) ) + var exprLoc: TLoc = initLocExpr(p, e[0][1]) + var tmp = getTemp(p, e.typ, needsInit=false) + putIntoDest(p, tmp, e, cCast(getTypeDesc(p.module, e.typ), addrLoc(p.config, exprLoc))) + putIntoDest(p, d, e, rdLoc(tmp)) else: putIntoDest(p, d, e, addrLoc(p.config, a), a.storage) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 0feb04d6c9..ee3e4adcac 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -63,10 +63,10 @@ # `i` should exception be raised in state `i`. For all states in `try` block # the target state is `except` block. For all states in `except` block # the target state is `finally` block. For all other states there is no -# target state (0, as the first block can never be neither except nor finally). +# target state (0, as the first state can never be except nor finally). # - env var :curExcLevel is created, finallies use it to decide their exit logic # - if there are finallies, env var :finallyPath is created. It contains exit state labels -# for every finally level, and is changed in runtime in try, except, break, and continue +# for every finally level, and is changed in runtime in try, except, break, and return # nodes to control finally exit behavior. # - the iter body is wrapped into a # var :tmp: Exception @@ -133,7 +133,7 @@ # of 3: # somethingElse() # :state = -1 # Exit -# break :staleLoop +# break :stateLoop # else: # return @@ -172,7 +172,7 @@ type stateLoopLabel: PSym # Label to break on, when jumping between states. tempVarId: int # unique name counter hasExceptions: bool # Does closure have yield in try? - curExcLandingState: PNode # Negative for except, positive for finally + curExcLandingState: PNode curFinallyLevel: int idgen: IdGenerator varStates: Table[ItemId, int] # Used to detect if local variable belongs to multiple states diff --git a/compiler/concepts.nim b/compiler/concepts.nim index 7c64b5eae9..a16b2fbfa2 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -271,7 +271,10 @@ proc matchType(c: PContext; fo, ao: PType; m: var MatchCon): bool = var a = ao f = fo - + if a.isSelf: + if m.magic in {mArrPut, mArrGet}: + return false + a = m.potentialImplementation if a.kind in bindableTypes: a = existingBinding(m, ao) if a == ao and a.kind == tyGenericParam and a.hasElementType and a.elementType.kind != tyNone: @@ -337,8 +340,11 @@ proc matchType(c: PContext; fo, ao: PType; m: var MatchCon): bool = result = true else: let ak = a.skipTypes(ignorableForArgType - {f.kind}) - if ak.kind == f.kind and f.kidsLen == ak.kidsLen: - result = matchKids(c, f, ak, m) + if ak.kind == f.kind: + if f.base.kind == tyNone: + result = true + elif f.kidsLen == ak.kidsLen: + result = matchKids(c, f, ak, m) of tyGenericInvocation, tyGenericInst: result = false let ea = a.skipTypes(ignorableForArgType) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index f57d451c06..ddb0f80bf4 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -100,6 +100,7 @@ when false: proc isLastReadImpl(n: PNode; c: var Con; scope: var Scope): bool = let root = parampatterns.exprRoot(n, allowCalls=false) if root == nil: return false + elif sfSingleUsedTemp in root.flags: return true var s = addr(scope) while s != nil: @@ -167,8 +168,7 @@ proc isLastRead(n: PNode; c: var Con; s: var Scope): bool = if not hasDestructor(c, n.typ) and (n.typ.kind != tyObject or isTrival(getAttachedOp(c.graph, n.typ, attachedAsgn))): return true let m = skipConvDfa(n) - result = (m.kind == nkSym and sfSingleUsedTemp in m.sym.flags) or - isLastReadImpl(n, c, s) + result = isLastReadImpl(n, c, s) proc isFirstWrite(n: PNode; c: var Con): bool = let m = skipConvDfa(n) @@ -1140,6 +1140,25 @@ proc genFieldAccessSideEffects(c: var Con; s: var Scope; dest, ri: PNode; flags: var snk = c.genSink(s, dest, newAccess, flags) result = newTree(nkStmtList, v, snk, c.genWasMoved(newAccess)) +proc ownsData(c: var Con; s: var Scope; orig: PNode; flags: set[MoveOrCopyFlag]): PNode = + var n = orig + while true: + case n.kind + of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr: + n = n[0] + else: + break + if n.kind in nkCallKinds and n.typ != nil and hasDestructor(c, n.typ): + result = newNodeIT(nkStmtListExpr, orig.info, orig.typ) + let tmp = c.getTemp(s, n.typ, n.info) + tmp.sym.flags.incl sfSingleUsedTemp + result.add newTree(nkFastAsgn, tmp, copyTree(n)) + s.final.add c.genDestroy(tmp) + n[] = tmp[] + result.add copyTree(orig) + else: + result = nil + proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopyFlag] = {}): PNode = var ri = ri var isEnsureMove = 0 @@ -1226,7 +1245,11 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy of nkRaiseStmt: result = pRaiseStmt(ri, c, s) else: - if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c, s) and + let isOwnsData = ownsData(c, s, ri2, flags) + + if isOwnsData != nil: + result = moveOrCopy(dest, isOwnsData, c, s, flags) + elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c, s) and canBeMoved(c, dest.typ): # Rule 3: `=sink`(x, z); wasMoved(z) let snk = c.genSink(s, dest, ri, flags) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index a9eb0263e9..03dda18caa 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -218,23 +218,38 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool, fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false) c.filterDiscriminator = oldfilterDiscriminator of nkRecList: - for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp, enforceWasMoved) + # destroys in reverse order #24719 + if c.kind == attachedDestructor: + for i in countdown(n.len-1, 0): + fillBodyObj(c, n[i], body, x, y, enforceDefaultOp, enforceWasMoved) + else: + for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp, enforceWasMoved) else: illFormedAstLocal(n, c.g.config) proc fillBodyObjTImpl(c: var TLiftCtx; t: PType, body, x, y: PNode) = - if t.baseClass != nil: - let dest = newNodeIT(nkHiddenSubConv, c.info, t.baseClass) - dest.add newNodeI(nkEmpty, c.info) - dest.add x - var src = y - if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}: - src = newNodeIT(nkHiddenSubConv, c.info, t.baseClass) - src.add newNodeI(nkEmpty, c.info) - src.add y + template fillBase = + if t.baseClass != nil: + let dest = newNodeIT(nkHiddenSubConv, c.info, t.baseClass) + dest.add newNodeI(nkEmpty, c.info) + dest.add x + var src = y + if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}: + src = newNodeIT(nkHiddenSubConv, c.info, t.baseClass) + src.add newNodeI(nkEmpty, c.info) + src.add y - fillBody(c, skipTypes(t.baseClass, abstractPtrs), body, dest, src) - fillBodyObj(c, t.n, body, x, y, enforceDefaultOp = false) + fillBody(c, skipTypes(t.baseClass, abstractPtrs), body, dest, src) + template fillFields = + fillBodyObj(c, t.n, body, x, y, enforceDefaultOp = false) + + if c.kind == attachedDestructor: + # destroys in reverse order #24719 + fillFields() + fillBase() + else: + fillBase() + fillFields() proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) = var hasCase = isCaseObj(t.n) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 65d216b831..7707d1a248 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -379,7 +379,9 @@ proc useVar(a: PEffects, n: PNode) = # If the variable is explicitly marked as .noinit. do not emit any error a.init.add s.id elif s.id notin a.init: - if s.typ.requiresInit: + if s.kind == skResult and tfRequiresInit in s.typ.flags: + localError(a.config, n.info, "'result' requires explicit initialization") + elif s.typ.requiresInit: message(a.config, n.info, warnProveInit, s.name.s) elif a.leftPartOfAsgn <= 0: if strictDefs in a.c.features: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index ce8b59f9cd..6c8542b584 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -725,7 +725,7 @@ template isLocalSym(sym: PSym): bool = sym.typ.kind == tyTypeDesc or sfCompileTime in sym.flags) or sym.kind in {skProc, skFunc, skIterator} and - sfGlobal notin sym.flags + sfGlobal notin sym.flags and sym.typ.callConv == ccClosure proc usesLocalVar(n: PNode): bool = case n.kind diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 42efcd2399..9d660a3bec 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1147,7 +1147,9 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType = let t = newTypeS(tySink, c, result) result = t else: discard - if result.kind == tyRef and c.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}: + if result.kind == tyRef and + c.config.selectedGC in {gcArc, gcOrc, gcAtomicArc} and + tfTriggersCompileTime notin result.flags: result.flags.incl tfHasAsgn proc findEnforcedStaticType(t: PType): PType = diff --git a/compiler/transf.nim b/compiler/transf.nim index a2090af841..197e073477 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -259,7 +259,8 @@ proc transformBlock(c: PTransf, n: PNode): PNode = var labl: PSym if c.inlining > 0: labl = newLabel(c, n[0]) - c.transCon.mapping[n[0].sym.itemId] = newSymNode(labl) + if n[0].kind != nkEmpty: + c.transCon.mapping[n[0].sym.itemId] = newSymNode(labl) else: labl = if n[0].kind != nkEmpty: @@ -552,7 +553,34 @@ proc transformConv(c: PTransf, n: PNode): PNode = # we don't include uint and uint64 here as these are no ordinal types ;-) if not isOrdinalType(source): # float -> int conversions. ugh. - result = transformSons(c, n) + # generate a range check: + if dest.kind in tyInt..tyInt64: + if dest.kind == tyInt64 or source.kind == tyInt64: + result = newTransNode(nkChckRange64, n, 3) + else: + result = newTransNode(nkChckRange, n, 3) + dest = skipTypes(n.typ, abstractVar) + + if dest.size < source.size: + let intType = + if source.size == 4: + getSysType(c.graph, n.info, tyInt32) + else: + getSysType(c.graph, n.info, tyInt64) + result[0] = + newTreeIT(n.kind, n.info, n.typ, n[0], + newTreeIT(nkConv, n.info, intType, + newNodeIT(nkType, n.info, intType), transform(c, n[1])) + ) + + else: + result[0] = transformSons(c, n) + + result[1] = newIntTypeNode(firstOrd(c.graph.config, dest), dest) + result[2] = newIntTypeNode(lastOrd(c.graph.config, dest), dest) + else: + result = transformSons(c, n) + elif firstOrd(c.graph.config, n.typ) <= firstOrd(c.graph.config, n[1].typ) and lastOrd(c.graph.config, n[1].typ) <= lastOrd(c.graph.config, n.typ): # BUGFIX: simply leave n as it is; we need a nkConv node, @@ -800,12 +828,20 @@ proc transformFor(c: PTransf, n: PNode): PNode = t = formal.ast.typ # better use the type that actually has a destructor. elif t.destructor == nil and arg.typ.destructor != nil: t = arg.typ - # generate a temporary and produce an assignment statement: - var temp = newTemp(c, t, formal.info) - #incl(temp.sym.flags, sfCursor) - addVar(v, temp) - stmtList.add(newAsgnStmt(c, nkFastAsgn, temp, arg, true)) - newC.mapping[formal.itemId] = temp + + if arg.kind in {nkDerefExpr, nkHiddenDeref}: + # optimizes for `[]` # bug #24093 + var temp = newTemp(c, arg[0].typ, formal.info) + addVar(v, temp) + stmtList.add(newAsgnStmt(c, nkFastAsgn, temp, arg[0], true)) + newC.mapping[formal.itemId] = newDeref(temp) + else: + # generate a temporary and produce an assignment statement: + var temp = newTemp(c, t, formal.info) + #incl(temp.sym.flags, sfCursor) + addVar(v, temp) + stmtList.add(newAsgnStmt(c, nkFastAsgn, temp, arg, true)) + newC.mapping[formal.itemId] = temp of paVarAsgn: assert(skipTypes(formal.typ, abstractInst).kind in {tyVar, tyLent}) newC.mapping[formal.itemId] = arg diff --git a/compiler/treetab.nim b/compiler/treetab.nim index 6685c4a899..1fd539f0f2 100644 --- a/compiler/treetab.nim +++ b/compiler/treetab.nim @@ -42,32 +42,37 @@ proc hashTree*(n: PNode): Hash = #echo "hashTree ", result #echo n -proc treesEquivalent(a, b: PNode): bool = +proc treesEquivalent(a, b: PNode; ignoreTypes: bool): bool = if a == b: result = true elif (a != nil) and (b != nil) and (a.kind == b.kind): case a.kind - of nkEmpty, nkNilLit, nkType: result = true + of nkEmpty: result = true of nkSym: result = a.sym.id == b.sym.id of nkIdent: result = a.ident.id == b.ident.id of nkCharLit..nkUInt64Lit: result = a.intVal == b.intVal - of nkFloatLit..nkFloat64Lit: result = a.floatVal == b.floatVal + of nkFloatLit..nkFloat64Lit: + result = cast[uint64](a.floatVal) == cast[uint64](b.floatVal) + #result = a.floatVal == b.floatVal of nkStrLit..nkTripleStrLit: result = a.strVal == b.strVal + of nkType, nkNilLit: + result = a.typ == b.typ else: if a.len == b.len: for i in 0..= 0: @@ -1182,8 +1189,10 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}, m: TMag of mEqF64: genBinaryABC(c, n, dest, opcEqFloat) of mLeF64: genBinaryABC(c, n, dest, opcLeFloat) of mLtF64: genBinaryABC(c, n, dest, opcLtFloat) - of mLePtr, mLeU: genBinaryABC(c, n, dest, opcLeu) - of mLtPtr, mLtU: genBinaryABC(c, n, dest, opcLtu) + of mLeU: genBinaryABC(c, n, dest, opcLeu) + of mLtU: genBinaryABC(c, n, dest, opcLtu) + of mLePtr, mLtPtr: + globalError(c.config, n.info, "pointer comparisons are not available at compile-time") of mEqProc, mEqRef: genBinaryABC(c, n, dest, opcEqRef) of mXor: genBinaryABC(c, n, dest, opcXor) @@ -1547,8 +1556,7 @@ template cannotEval(c: PCtx; n: PNode) = if c.config.cmd == cmdCheck and c.config.m.errorOutputs != {}: # nim check command with no error outputs doesn't need to cascade here, # includes `tryConstExpr` case which should not continue generating code - localError(c.config, n.info, "cannot evaluate at compile time: " & - n.renderTree) + localError(c.config, n.info, "cannot evaluate at compile time: " & n.renderTree) c.cannotEval = true return globalError(c.config, n.info, "cannot evaluate at compile time: " & @@ -1886,7 +1894,7 @@ proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = c.freeTemp(objR) proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = - if n[0].typ == nil: + if n[0].typ == nil: globalError(c.config, n.info, "cannot access array with nil type") return diff --git a/doc/manual.md b/doc/manual.md index 9abd0e762c..29006a7536 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -4062,7 +4062,7 @@ notation. (Thus an operator can have more than two parameters): # Multiply and add result = a * b + c - assert `*+`(3, 4, 6) == `+`(`*`(a, b), c) + assert `*+`(3, 4, 6) == `+`(`*`(3, 4), 6) ``` diff --git a/doc/nimc.md b/doc/nimc.md index 1bc3edc4cf..24a2b3afe1 100644 --- a/doc/nimc.md +++ b/doc/nimc.md @@ -224,6 +224,8 @@ directories (in this order; later files overwrite previous settings): command-line option. +[NimScript files](nims.html) can also be used for configuration. + Command-line settings have priority over configuration file settings. The default build of a project is a `debug build`:idx:. To compile a diff --git a/doc/tut2.md b/doc/tut2.md index 1b59288d5b..94434a84c8 100644 --- a/doc/tut2.md +++ b/doc/tut2.md @@ -86,6 +86,26 @@ Student(id: 123)` will truncate subclass fields. (*is-a* relation) for simple code reuse. Since objects are value types in Nim, composition is as efficient as inheritance. +Interfaces +---------- +Concepts like abstract classes, protocols, traits, and interfaces can be +simulated as objects of closures: + +```nim + +type + IntFieldInterface = object + getter: proc (): int + setter: proc (x: int) + + +proc outer: IntFieldInterface = + var captureMe = 0 + proc getter(): int = result = captureMe + proc setter(x: int) = captureMe = x + + result = IntFieldInterface(getter: getter, setter: setter) +``` Mutually recursive types ------------------------ diff --git a/koch.nim b/koch.nim index dc5ad649c4..48cd2cdcb2 100644 --- a/koch.nim +++ b/koch.nim @@ -11,9 +11,9 @@ const # examples of possible values for repos: Head, ea82b54 - NimbleStableCommit = "6a2486b597132340ea7422b078c769b58f21d16d" # 0.20.0 + NimbleStableCommit = "9207e8b2bbdf66b5a4d1020214cff44d2d30df92" # 0.20.1 AtlasStableCommit = "26cecf4d0cc038d5422fc1aa737eec9c8803a82b" # 0.9 - ChecksumsStableCommit = "f8f6bd34bfa3fe12c64b919059ad856a96efcba0" # 2.0.1 + ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1 SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01" NimonyStableCommit = "1dbabac403ae32e185ee4c29f006d04e04b50c6d" # unversioned \ diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 3825904781..3a57845c22 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -428,7 +428,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = dealloc buffer buffer = nil ) - ol.offset = DWORD(f.offset and 0xffffffff) + ol.offset = cast[DWORD](f.offset and 0xffffffff) ol.offsetHigh = DWORD(f.offset shr 32) f.offset.inc(data.len) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 42d54c8392..3dba087aa8 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -116,6 +116,11 @@ macro evalOnceAs(expAlias, exp: untyped, newProc(name = genSym(nskTemplate, $expAlias), params = [getType(untyped)], body = val, procType = nnkTemplateDef)) +template unCheckedInc(x) = + {.push overflowChecks: off.} + inc(x) + {.pop.} + func concat*[T](seqs: varargs[seq[T]]): seq[T] = ## Takes several sequences' items and returns them inside a new sequence. ## All sequences must be of the same type. @@ -139,7 +144,7 @@ func concat*[T](seqs: varargs[seq[T]]): seq[T] = for s in items(seqs): for itm in items(s): result[i] = itm - inc(i) + unCheckedInc(i) func addUnique*[T](s: var seq[T], x: sink T) = ## Adds `x` to the container `s` if it is not already present. @@ -170,7 +175,7 @@ func count*[T](s: openArray[T], x: T): int = result = 0 for itm in items(s): if itm == x: - inc result + unCheckedInc result func cycle*[T](s: openArray[T], n: Natural): seq[T] = ## Returns a new sequence with the items of the container `s` repeated @@ -188,7 +193,7 @@ func cycle*[T](s: openArray[T], n: Natural): seq[T] = for x in 0 ..< n: for e in s: result[o] = e - inc o + unCheckedInc o proc repeat*[T](x: T, n: Natural): seq[T] = ## Returns a new sequence with the item `x` repeated `n` times. @@ -321,6 +326,26 @@ func minmax*[T](x: openArray[T], cmp: proc(a, b: T): int): (T, T) {.effectsOf: c elif cmp(result[1], x[i]) < 0: result[1] = x[i] +template findIt*(s, predicate: untyped): int = + ## Iterates through a container and returns the index of the first item that + ## fulfills the predicate, or -1 + ## + ## Unlike the `find`, the predicate needs to be an expression using + ## the `it` variable for testing, like: `findIt([3, 2, 1], it == 2)`. + var + res = -1 + i = 0 + + # We must use items here since both `find` and `anyIt` are defined in terms + # of `items` + # (and not `pairs`) + for it {.inject.} in items(s): + if predicate: + res = i + break + unCheckedInc(i) + res + template zipImpl(s1, s2, retType: untyped): untyped = proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = ## Returns a new sequence with a combination of the two input containers. @@ -417,7 +442,7 @@ func distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = if extra == 0 or spread == false: # Use an algorithm which overcounts the stride and minimizes reading limits. - if extra > 0: inc(stride) + if extra > 0: unCheckedInc(stride) for i in 0 ..< num: result[i] = newSeq[T]() for g in first ..< min(s.len, first + stride): @@ -429,7 +454,7 @@ func distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = last = first + stride if extra > 0: extra -= 1 - inc(last) + unCheckedInc(last) result[i] = newSeq[T]() for g in first ..< last: result[i].add(s[g]) @@ -586,7 +611,7 @@ proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) s[pos] = move(s[i]) else: shallowCopy(s[pos], s[i]) - inc(pos) + unCheckedInc(pos) setLen(s, pos) func delete*[T](s: var seq[T]; slice: Slice[int]) = @@ -617,8 +642,8 @@ func delete*[T](s: var seq[T]; slice: Slice[int]) = s[i] = move(s[j]) else: s[i].shallowCopy(s[j]) - inc(i) - inc(j) + unCheckedInc(i) + unCheckedInc(j) setLen(s, newLen) when nimvm: defaultImpl() else: @@ -649,8 +674,8 @@ func delete*[T](s: var seq[T]; first, last: Natural) {.deprecated: "use `delete( s[i] = move(s[j]) else: s[i].shallowCopy(s[j]) - inc(i) - inc(j) + unCheckedInc(i) + unCheckedInc(j) setLen(s, newLen) func insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = @@ -681,10 +706,10 @@ func insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = dec(i) dec(j) # Insert items from `dest` into `dest` at `pos` - inc(j) + unCheckedInc(j) for item in src: dest[j] = item - inc(j) + unCheckedInc(j) template filterIt*(s, pred: untyped): untyped = @@ -715,7 +740,7 @@ template filterIt*(s, pred: untyped): untyped = var result = newSeq[typeof(s[0])]() for it {.inject.} in items(s): if pred: result.add(it) - result + move result template keepItIf*(varSeq: seq, pred: untyped) = ## Keeps the items in the passed sequence (must be declared as a `var`) @@ -743,7 +768,7 @@ template keepItIf*(varSeq: seq, pred: untyped) = varSeq[pos] = move(varSeq[i]) else: shallowCopy(varSeq[pos], varSeq[i]) - inc(pos) + unCheckedInc(pos) setLen(varSeq, pos) since (1, 1): @@ -842,12 +867,7 @@ template anyIt*(s, pred: untyped): bool = assert numbers.anyIt(it > 8) == true assert numbers.anyIt(it > 9) == false - var result = false - for it {.inject.} in items(s): - if pred: - result = true - break - result + findIt(s, pred) != -1 template toSeq1(s: not iterator): untyped = # overload for typed but not iterator @@ -875,7 +895,7 @@ template toSeq2(iter: iterator): untyped = var result = newSeq[typeof(iter2)](iter2.len) for x in iter2: result[i] = x - inc i + unCheckedInc i result else: type OutType = typeof(iter2()) @@ -920,7 +940,7 @@ template toSeq*(iter: untyped): untyped = var i = 0 for x in iter2: result[i] = x - inc i + unCheckedInc i result else: var result: seq[typeof(iter)] = @[] diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 92f85b1464..94d8721b96 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -707,7 +707,7 @@ template withValue*[A, B](t: Table[A, B], key: A, mixin rawGet var hc: Hash var index = rawGet(t, key, hc) - if index > 0: + if index >= 0: let value {.cursor, inject.} = t.data[index].val body1 else: diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index ea70ed0c3c..ff6fcb3a66 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -220,7 +220,16 @@ ## ```Nim ## import std/httpclient ## -## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let myProxy = newProxy("http://user:password@myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## SOCKS5 proxy with proxy-side DNS resolving: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("socks5h://user:password@myproxy.network") ## let client = newHttpClient(proxy = myProxy) ## ``` ## @@ -338,7 +347,6 @@ proc body*(response: AsyncResponse): Future[string] {.async.} = type Proxy* = ref object url*: Uri - auth*: string MultipartEntry = object name, content: string @@ -387,13 +395,30 @@ proc getDefaultSSL(): SslContext = result = defaultSslContext doAssert result != nil, "failure to initialize the SSL context" -proc newProxy*(url: string; auth = ""): Proxy = +proc newProxy*(url: Uri): Proxy = ## Constructs a new `TProxy` object. - result = Proxy(url: parseUri(url), auth: auth) + result = Proxy(url: url) -proc newProxy*(url: Uri; auth = ""): Proxy = +proc newProxy*(url: string): Proxy = ## Constructs a new `TProxy` object. - result = Proxy(url: url, auth: auth) + result = Proxy(url: parseUri(url)) + +proc newProxy*(url: Uri; auth: string): Proxy {.deprecated: "Provide auth in url instead".} = + result = Proxy(url: url) + if auth != "": + let parts = auth.split(':') + if parts.len != 2: + raise newException(ValueError, "Invalid auth string") + result.url.username = parts[0] + result.url.password = parts[1] + +proc newProxy*(url: string; auth: string): Proxy {.deprecated: "Provide auth in url instead".} = + result = newProxy(parseUri(url), auth) + +proc auth*(p: Proxy): string {.deprecated: "Get auth from p.url.username and p.url.password".} = + result = "" + if p.url.username != "" or p.url.password != "": + result = p.url.username & ":" & p.url.password proc newMultipartData*: MultipartData {.inline.} = ## Constructs a new `MultipartData` object. @@ -548,7 +573,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade result = $httpMethod result.add ' ' - if proxy.isNil or requestUrl.scheme == "https": + if proxy.isNil or (requestUrl.scheme == "https" and proxy.url.scheme == "socks5h"): # /path?query if not requestUrl.path.startsWith("/"): result.add '/' result.add(requestUrl.path) @@ -575,8 +600,8 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade add(result, "Connection: Keep-Alive" & httpNewLine) # Proxy auth header. - if not proxy.isNil and proxy.auth != "": - let auth = base64.encode(proxy.auth) + if not proxy.isNil and proxy.url.username != "": + let auth = base64.encode(proxy.url.username & ":" & proxy.url.password) add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) for key, val in headers: @@ -689,7 +714,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, let exampleHtml = waitFor asyncProc() assert "Example Domain" in exampleHtml assert "Pizza" notin exampleHtml - + new result result.headers = headers result.userAgent = userAgent @@ -941,17 +966,75 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, when client is AsyncHttpClient: result.bodyStream.complete() +proc startSsl(client: HttpClient | AsyncHttpClient, hostname: string) = + when defined(ssl): + try: + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, hostname) + except: + client.socket.close() + raise getCurrentException() + +proc socks5hHandshake(client: HttpClient | AsyncHttpClient, + url: Uri) {.multisync.} = + var hasAuth = client.proxy.url.username != "" + if hasAuth: + await client.socket.send("\x05\x02\x00\x02") # Propose auth + else: + await client.socket.send("\x05\x01\x00") # Connect with no auth + + when client.socket is Socket: + var resp = client.socket.recv(2, client.timeout) + else: + var resp = await client.socket.recv(2) + + if resp == "\x05\x02" and hasAuth: + # Perform auth + let authStr = "\x01" & + char(client.proxy.url.username.len) & client.proxy.url.username & + char(client.proxy.url.password.len) & client.proxy.url.password + await client.socket.send(authStr) + when client.socket is Socket: + resp = client.socket.recv(2, client.timeout) + else: + resp = await client.socket.recv(2) + if resp != "\x01\x00": + httpError("Proxy authentication failed") + elif resp != "\x05\x00": + httpError("Unexpected proxy response: " & resp.toHex()) + + let port = if url.port != "": parseInt(url.port) + elif url.scheme == "http": 80 + else: 443 + var p = " " + p[0] = cast[char](port.uint16 shr 8) + p[1] = cast[char](port) + await client.socket.send("\x05\x01\x00\x03" & url.hostname.len.char & url.hostname & p) + when client.socket is Socket: + resp = client.socket.recv(10, client.timeout) + else: + resp = await client.socket.recv(10) + if resp.len != 10 or resp[0] != '\x05' or resp[1] != '\x00': + httpError("Unexpected proxy response: " & resp.toHex()) + proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = if client.currentURL.hostname != url.hostname or client.currentURL.scheme != url.scheme or client.currentURL.port != url.port or (not client.connected): - # Connect to proxy if specified - let connectionUrl = - if client.proxy.isNil: url else: client.proxy.url - let isSsl = connectionUrl.scheme.toLowerAscii() == "https" + var isSsl = false + var connectionUrl = url + if client.proxy.isNil: + isSsl = url.scheme.toLowerAscii() == "https" + else: + connectionUrl = client.proxy.url + let proxyScheme = connectionUrl.scheme.toLowerAscii() + if proxyScheme == "https": + isSsl = true + elif proxyScheme == "socks5h": + isSsl = url.scheme.toLowerAscii() == "https" if isSsl and not defined(ssl): raise newException(HttpRequestError, @@ -976,37 +1059,33 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.socket = await asyncnet.dial(connectionUrl.hostname, port) else: {.fatal: "Unsupported client type".} - when defined(ssl): - if isSsl: - try: + if not client.proxy.isNil and client.proxy.url.scheme.toLowerAscii() == "socks5h": + await socks5hHandshake(client, url) + if isSsl: startSsl(client, url.hostname) + else: + if isSsl: startSsl(client, connectionUrl.hostname) + # If need to CONNECT through http(s) proxy + if url.scheme == "https" and not client.proxy.isNil: + when defined(ssl): + # Pass only host:port for CONNECT + var connectUrl = initUri() + connectUrl.hostname = url.hostname + connectUrl.port = if url.port != "": url.port else: "443" + + let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, + newHttpHeaders(), client.proxy) + await client.socket.send(proxyHeaderString) + let proxyResp = await parseResponse(client, false) + + if not proxyResp.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") client.sslContext.wrapConnectedSocket( - client.socket, handshakeAsClient, connectionUrl.hostname) - except: - client.socket.close() - raise getCurrentException() - - # If need to CONNECT through proxy - if url.scheme == "https" and not client.proxy.isNil: - when defined(ssl): - # Pass only host:port for CONNECT - var connectUrl = initUri() - connectUrl.hostname = url.hostname - connectUrl.port = if url.port != "": url.port else: "443" - - let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, - newHttpHeaders(), client.proxy) - await client.socket.send(proxyHeaderString) - let proxyResp = await parseResponse(client, false) - - if not proxyResp.status.startsWith("200"): + client.socket, handshakeAsClient, url.hostname) + else: raise newException(HttpRequestError, - "The proxy server rejected a CONNECT request, " & - "so a secure connection could not be established.") - client.sslContext.wrapConnectedSocket( - client.socket, handshakeAsClient, url.hostname) - else: - raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") # May be connected through proxy but remember actual URL being accessed client.currentURL = url @@ -1086,7 +1165,7 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, var data: seq[string] = @[] if multipart != nil and multipart.content.len > 0: - # `format` modifies `client.headers`, see + # `format` modifies `client.headers`, see # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 data = await client.format(multipart) newHeaders = client.headers.override(headers) @@ -1319,7 +1398,7 @@ proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) = defer: client.getBody = true let resp = client.get(url) - + if resp.code.is4xx or resp.code.is5xx: raise newException(HttpRequestError, resp.status) @@ -1334,7 +1413,7 @@ proc downloadFileEx(client: AsyncHttpClient, ## Downloads `url` and saves it to `filename`. client.getBody = false let resp = await client.get(url) - + if resp.code.is4xx or resp.code.is5xx: raise newException(HttpRequestError, resp.status) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 7bc6fcdcee..c02b6dddc6 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -12,6 +12,8 @@ ## ## Unstable API. +{.deprecated: "use `nimble install parsesql` and import `pkg/parsesql` instead".} + import std/[strutils, lexbase] import std/private/decode_helpers diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index f4ea498880..9391c391ca 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -25,6 +25,10 @@ ## Solaris (files, sockets, handles and user events). ## Android (files, sockets, handles and user events). ## +## By default, the implementation is chosen based on the target +## platform; you can pass `-d:nimIoselector=value` to override it. +## Accepted values are "epoll", "kqueue", "poll", and "select". +## ## TODO: `/dev/poll`, `event ports` and filesystem events. import std/nativesockets @@ -342,7 +346,9 @@ else: res = int(fdLim.rlim_cur) - 1 res - when defined(nimIoselector): + const nimIoselector {.strdefine.} = "" + + when nimIoselector != "": when nimIoselector == "epoll": include ioselects/ioselectors_epoll elif nimIoselector == "kqueue": diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index c218ac1c53..1a5d4d5d6f 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -74,6 +74,7 @@ import std/parseutils from std/math import pow, floor, log10 from std/algorithm import fill, reverse import std/enumutils +from std/bitops import fastLog2 from std/unicode import toLower, toUpper export toLower, toUpper @@ -2639,37 +2640,35 @@ func formatSize*(bytes: int64, ## * `strformat module`_ for string interpolation and formatting runnableExamples: doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" - doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize((2.234*1024*1024).int) == "2.233MiB" doAssert formatSize(4096, includeSpace = true) == "4 KiB" doAssert formatSize(4096, prefix = bpColloquial, includeSpace = true) == "4 kB" doAssert formatSize(4096) == "4KiB" - doAssert formatSize(5_378_934, prefix = bpColloquial, decimalSep = ',') == "5,13MB" + doAssert formatSize(5_378_934, prefix = bpColloquial, decimalSep = ',') == "5,129MB" - const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] - const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] - var - xb: int64 = bytes - fbytes: float - lastXb: int64 = bytes - matchedIndex = 0 - prefixes: array[9, string] + # It doesn't needs Zi and larger units until we use int72 or larger ints. + const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei"] + const collPrefixes = ["", "k", "M", "G", "T", "P", "E"] + + let lg2 = if bytes == 0: + 0 + else: + when hasWorkingInt64: + fastLog2(bytes) + else: + fastLog2(int32 bytes) + let matchedIndex = lg2 div 10 + # Lower bits that are smaller than 0.001 when `bytes` is converted to a real number and added prefix, are discard. + # Then it is converted to float with round down. + let discardBits = (lg2 div 10 - 1) * 10 + + var prefixes: array[7, string] if prefix == bpColloquial: prefixes = collPrefixes else: prefixes = iecPrefixes - # Iterate through prefixes seeing if value will be greater than - # 0 in each case - for index in 1..`_ osdirs.setCurrentDir(newDir.string) + +proc copyDir*(source, dest: Path; skipSpecial = false) {.inline, + tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect].} = + ## Copies a directory from `source` to `dest`. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will copy the attributes from + ## `source` into `dest`. + ## + ## On other platforms created files and directories will inherit the + ## default permissions of a newly created file/directory for the user. + ## Use `copyDirWithPermissions proc`_ + ## to preserve attributes recursively on these platforms. + ## + ## See also: + ## * `copyDirWithPermissions proc`_ + copyDir(source.string, dest.string, skipSpecial) + +proc copyDirWithPermissions*(source, dest: Path; + ignorePermissionErrors = true, + skipSpecial = false) + {.inline, tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect].} = + ## Copies a directory from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs + ## on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file or directory + ## itself has been copied, which won't happen atomically and could lead to a + ## race condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyDir proc`_ + copyDirWithPermissions(source.string, dest.string, + ignorePermissionErrors, skipSpecial) diff --git a/lib/std/files.nim b/lib/std/files.nim index c4e0491c99..223e51b6cc 100644 --- a/lib/std/files.nim +++ b/lib/std/files.nim @@ -6,8 +6,43 @@ from std/paths import Path, ReadDirEffect, WriteDirEffect from std/private/osfiles import fileExists, removeFile, - moveFile + moveFile, copyFile, copyFileWithPermissions, + copyFileToDir, tryRemoveFile, + getFilePermissions, setFilePermissions, + CopyFlag, FilePermission +export CopyFlag, FilePermission + + +proc getFilePermissions*(filename: Path): set[FilePermission] {.inline, tags: [ReadDirEffect].} = + ## Retrieves file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is checked, every other + ## permission is available in any case. + ## + ## See also: + ## * `setFilePermissions proc`_ + result = getFilePermissions(filename.string) + +proc setFilePermissions*(filename: Path, permissions: set[FilePermission], + followSymlinks = true) + {.inline, tags: [ReadDirEffect, WriteDirEffect].} = + ## Sets the file permissions for `filename`. + ## + ## If `followSymlinks` set to true (default) and ``filename`` points to a + ## symlink, permissions are set to the file symlink points to. + ## `followSymlinks` set to false is a noop on Windows and some POSIX + ## systems (including Linux) on which `lchmod` is either unavailable or always + ## fails, given that symlinks permissions there are not observed. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is changed, depending on + ## ``fpUserWrite`` permission. + ## + ## See also: + ## * `getFilePermissions proc`_ + setFilePermissions(filename.string, permissions, followSymlinks) proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = ## Returns true if `filename` exists and is a regular file or symlink. @@ -15,6 +50,18 @@ proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect], sideEffe ## Directories, device files, named pipes and sockets return false. result = fileExists(filename.string) +proc tryRemoveFile*(file: Path): bool {.inline, tags: [WriteDirEffect].} = + ## Removes the `file`. + ## + ## If this fails, returns `false`. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeFile proc`_ + result = tryRemoveFile(file.string) + proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} = ## Removes the `file`. ## @@ -26,6 +73,7 @@ proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} = ## See also: ## * `removeDir proc `_ ## * `moveFile proc`_ + ## * `tryRemoveFile proc`_ removeFile(file.string) proc moveFile*(source, dest: Path) {.inline, @@ -44,3 +92,73 @@ proc moveFile*(source, dest: Path) {.inline, ## * `moveDir proc `_ ## * `removeFile proc`_ moveFile(source.string, dest.string) + +proc copyFile*(source, dest: Path; options = cfSymlinkFollow; bufferSize = 16_384) {.inline, tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect].} = + ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will + ## copy the source file's attributes into dest. + ## + ## On other platforms you need + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ + ## procs + ## to copy them by hand (or use the convenience `copyFileWithPermissions + ## proc`_), + ## otherwise `dest` will inherit the default permissions of a newly + ## created file for the user. + ## + ## If `dest` already exists, the file attributes + ## will be preserved and the content overwritten. + ## + ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless + ## `-d:nimLegacyCopyFile` is used. + ## + ## `copyFile` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `copyFileWithPermissions proc`_ + copyFile(source.string, dest.string, {options}, bufferSize) + +proc copyFileWithPermissions*(source, dest: Path; + ignorePermissionErrors = true, + options = cfSymlinkFollow) {.inline.} = + ## Copies a file from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ + ## procs on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file itself has + ## been copied, which won't happen atomically and could lead to a race + ## condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyFile proc`_ + copyFileWithPermissions(source.string, dest.string, + ignorePermissionErrors, {options}) + +proc copyFileToDir*(source, dir: Path, options = cfSymlinkFollow; bufferSize = 16_384) {.inline.} = + ## Copies a file `source` into directory `dir`, which must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## `copyFileToDir` allows to specify `bufferSize` to improve I/O performance. + copyFileToDir(source.string, dir.string, {options}, bufferSize) diff --git a/lib/system.nim b/lib/system.nim index a18e81d3d7..fbbc2a3398 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1465,6 +1465,8 @@ proc isNil*[T: proc | iterator {.closure.}](x: T): bool {.noSideEffect, magic: " ## Fast check whether `x` is nil. This is sometimes more efficient than ## `== nil`. +proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} + when defined(nimHasTopDownInference): # magic used for seq type inference proc `@`*[T](a: openArray[T]): seq[T] {.magic: "OpenArrayToSeq".} = @@ -1472,8 +1474,17 @@ when defined(nimHasTopDownInference): ## ## This is not as efficient as turning a fixed length array into a sequence ## as it always copies every element of `a`. - newSeq(result, a.len) - for i in 0..a.len-1: result[i] = a[i] + let sz = a.len + when supportsCopyMem(T) and not defined(js): + result = newSeqUninit[T](sz) + when nimvm: + for i in 0..sz-1: result[i] = a[i] + else: + if sz != 0: + copyMem(addr result[0], addr a[0], sizeof(T) * sz) + else: + newSeq(result, sz) + for i in 0..sz-1: result[i] = a[i] else: proc `@`*[T](a: openArray[T]): seq[T] = ## Turns an *openArray* into a sequence. @@ -1644,8 +1655,6 @@ when not defined(js) and defined(nimV2): vTable: UncheckedArray[pointer] # vtable for types PNimTypeV2 = ptr TNimTypeV2 -proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} - when notJSnotNims and defined(nimSeqsV2): include "system/strs_v2" include "system/seqs_v2" diff --git a/tests/arc/tdestructor_order.nim b/tests/arc/tdestructor_order.nim new file mode 100644 index 0000000000..6b707486e6 --- /dev/null +++ b/tests/arc/tdestructor_order.nim @@ -0,0 +1,39 @@ +discard """ + output: ''' +destroying d +destroying c +destroying a 2 +destroying d +destroying c +destroying a 1 +''' +joinable: false +""" + +type + Aaaa {.inheritable.} = object + vvvv: int + Bbbb = object of Aaaa + c: Cccc + d: Dddd + Cccc = object + Dddd = object + + Holder = object + member: ref Aaaa + +proc `=destroy`(v: Cccc) = + echo "destroying c" + +proc `=destroy`(v: Dddd) = + echo "destroying d" + +proc `=destroy`(v: Aaaa) = + echo "destroying a ", v.vvvv + +func makeHolder(vvvv: int): ref Holder = + (ref Holder)(member: (ref Bbbb)(vvvv: vvvv)) + +block: + var v = makeHolder(1) + var v2 = makeHolder(2) \ No newline at end of file diff --git a/tests/ccgbugs/taddrconvs.nim b/tests/ccgbugs/taddrconvs.nim index 6990648c4a..759b1c1e0f 100644 --- a/tests/ccgbugs/taddrconvs.nim +++ b/tests/ccgbugs/taddrconvs.nim @@ -25,3 +25,23 @@ block: var m = uint64(12) foo(culonglong(m)) main() + +block: # bug #25109 + type T = culonglong + proc r(c: var T) = c = 1 + proc h(a: var culonglong) = r(T(a)) + var a: culonglong + h(a) + doAssert a == 1 + +block: # bug #25111 + type T = culonglong + proc r(c: var T) = c = 1 + + proc foo = + var a: uint64 + r(T(a)) + doAssert a == 1 + + foo() + diff --git a/tests/concepts/tconceptsv2.nim b/tests/concepts/tconceptsv2.nim index 369fd3e854..c735aeeacc 100644 --- a/tests/concepts/tconceptsv2.nim +++ b/tests/concepts/tconceptsv2.nim @@ -497,6 +497,28 @@ block: spring({One,Two}) +block: # bare `range` + type + MyRange = 0..64 + MyConcept = concept + proc a(x: typedesc[Self]) + + proc a(x: typedesc[range]) = discard + proc spring(x: typedesc[MyConcept]) = discard + spring(MyRange) + +block: + type + A = object + TestConcept = + concept + proc x(x: Self) + + proc x(x: not object) = + discard + + assert A isnot TestConcept + # this code fails inside a block for some reason type Indexable[T] = concept proc `[]`(t: Self, i: int): T diff --git a/tests/destructor/tdestructor.nim b/tests/destructor/tdestructor.nim index e081eb251d..bb5889f345 100644 --- a/tests/destructor/tdestructor.nim +++ b/tests/destructor/tdestructor.nim @@ -1,5 +1,6 @@ discard """ - output: '''----1 + output: ''' +----1 myobj constructed myobj destroyed ----2 @@ -14,8 +15,8 @@ mygeneric3 constructed mygeneric1 destroyed ----5 mydistinctObj constructed -myobj destroyed mygeneric2 destroyed +myobj destroyed ------------------8 mygeneric1 destroyed ----6 diff --git a/tests/destructor/tnewruntime_strutils.nim b/tests/destructor/tnewruntime_strutils.nim index 7e4074218f..e73e9b2445 100644 --- a/tests/destructor/tnewruntime_strutils.nim +++ b/tests/destructor/tnewruntime_strutils.nim @@ -53,11 +53,11 @@ proc nonStaticTests = block: # formatSize tests when not defined(js): doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" # <=== bug #8231 - doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize((2.234*1024*1024).int) == "2.233MiB" doAssert formatSize(4096) == "4KiB" doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" doAssert formatSize(4096, includeSpace=true) == "4 KiB" - doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,129MB" block: # formatEng tests doAssert formatEng(0, 2, trim=false) == "0.00" diff --git a/tests/errmsgs/t25117.nim b/tests/errmsgs/t25117.nim new file mode 100644 index 0000000000..1f63e65b49 --- /dev/null +++ b/tests/errmsgs/t25117.nim @@ -0,0 +1,13 @@ +discard """ + errormsg: "'result' requires explicit initialization" +""" + +type RI {.requiresInit.} = object + v: int + +proc xxx(v: var RI) = discard + +proc f(T: type): T = + xxx(result) # Should fail + +discard f(RI) \ No newline at end of file diff --git a/tests/errmsgs/t25120.nim b/tests/errmsgs/t25120.nim new file mode 100644 index 0000000000..217bba1fa7 --- /dev/null +++ b/tests/errmsgs/t25120.nim @@ -0,0 +1,6 @@ +discard """ + errormsg: "request to generate code for .compileTime proc: riesig" +""" + +proc riesig(): NimNode = discard +discard riesig() diff --git a/tests/global/tglobalclosure.nim b/tests/global/tglobalclosure.nim new file mode 100644 index 0000000000..81cdba37da --- /dev/null +++ b/tests/global/tglobalclosure.nim @@ -0,0 +1,14 @@ +discard """ + errormsg: "cannot assign local to global variable" + line: 11 +""" + +proc main() = + var x = "hi" + proc p() = + echo x + + let a {.global.} = p + p() + +main() diff --git a/tests/global/tglobalclosure2.nim b/tests/global/tglobalclosure2.nim new file mode 100644 index 0000000000..bd6b9bfd61 --- /dev/null +++ b/tests/global/tglobalclosure2.nim @@ -0,0 +1,17 @@ +discard """ + errormsg: "cannot assign local to global variable" + line: 14 +""" + +type X = object + p: proc() {.closure.} + +proc main() = + var x = "hi" + proc p() = + echo x + + let a {.global.} = X(p: p) + a.p() + +main() diff --git a/tests/global/tglobalproc.nim b/tests/global/tglobalproc.nim new file mode 100644 index 0000000000..ba6f5a7999 --- /dev/null +++ b/tests/global/tglobalproc.nim @@ -0,0 +1,17 @@ +discard """ +output: "hi\nhi" +""" + +type X = object + p: proc() {.nimcall.} + +proc main() = + proc p() = + echo "hi" + + let a {.global.} = p + let b {.global.} = X(p: p) + a() + b.p() + +main() diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index 2452102bd3..efea76c02e 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -412,3 +412,15 @@ block: # bug #24033 collections.add (id, str, $num) doAssert collections[1] == (1, "foo", "3.14") + + +block: # bug #25121 + iterator k(): int = + when nimvm: + yield 0 + else: + yield 0 + + for _ in k(): + (proc() = (; let _ = block: 0))() + diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim index 0bd4796704..99ccaba8b3 100644 --- a/tests/stdlib/thttpclient.nim +++ b/tests/stdlib/thttpclient.nim @@ -107,6 +107,13 @@ proc asyncTest() {.async.} = # client = newAsyncHttpClient(proxy = newProxy("http://51.254.106.76:80/")) # var resp = await client.request("https://github.com") # echo resp + # + # SOCKS5H proxy test + # when manualTests: + # block: + # client = newAsyncHttpClient(proxy = newProxy("socks5h://user:blabla@127.0.0.1:9050")) + # var resp = await client.request("https://api.my-ip.io/v2/ip.txt") + # echo await resp.body proc syncTest() = var client = newHttpClient() diff --git a/tests/stdlib/thttpclient_ssl.nim b/tests/stdlib/thttpclient_ssl.nim index 4208767925..962d047bb9 100644 --- a/tests/stdlib/thttpclient_ssl.nim +++ b/tests/stdlib/thttpclient_ssl.nim @@ -1,6 +1,6 @@ discard """ cmd: "nim $target --mm:refc -d:ssl $options $file" - disabled: "openbsd" + disabled: "true" retries: 2 """ diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index 611659fdbb..3b77ec3c0a 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -30,6 +30,9 @@ Raises from stdtest/specialpaths import buildDir import std/[syncio, assertions, osproc, os, strutils, pathnorm] +import std/paths except getCurrentDir +import std/[files, dirs] + block fileOperations: let files = @["these.txt", "are.x", "testing.r", "files.q"] let dirs = @["some", "created", "test", "dirs"] @@ -52,11 +55,11 @@ block fileOperations: doAssertRaises(OSError): copyFile(file, dname/sub/fname2) doAssertRaises(OSError): copyFileToDir(file, dname/sub) doAssertRaises(ValueError): copyFileToDir(file, "") - copyFile(file, file2) + copyFile(Path file, Path file2) doAssert fileExists(file2) doAssert readFile(file2) == str createDir(dname/sub) - copyFileToDir(file, dname/sub) + copyFileToDir(Path file, Path dname/sub) doAssert fileExists(dname/sub/fname) removeDir(dname/sub) doAssert not dirExists(dname/sub) @@ -131,12 +134,13 @@ block fileOperations: removeDir(dname) # test copyDir: - createDir("a/b") + createDir(Path "a/b") open("a/b/file.txt", fmWrite).close createDir("a/b/c") open("a/b/c/fileC.txt", fmWrite).close - copyDir("a", "../dest/a") + createDir(Path"a/b") + copyDir(Path "a", Path "../dest/a") removeDir("a") doAssert dirExists("../dest/a/b") @@ -169,7 +173,7 @@ block fileOperations: doAssert execCmd("mkfifo -m 600 a/fifoFile") == 0 copyDir("a/", "../dest/a/", skipSpecial = true) - copyDirWithPermissions("a/", "../dest2/a/", skipSpecial = true) + copyDirWithPermissions(Path "a/", Path "../dest2/a/", skipSpecial = true) removeDir("a") # Symlink handling in `copyFile`, `copyFileWithPermissions`, `copyFileToDir`, diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim index df0fb1610a..027f9b1195 100644 --- a/tests/stdlib/tsequtils.nim +++ b/tests/stdlib/tsequtils.nim @@ -258,6 +258,17 @@ block: # any doAssert any(anumbers, proc (x: int): bool = return x > 8) == true doAssert any(anumbers, proc (x: int): bool = return x > 9) == false +block: # findIt + let + numbers = @[1, 4, 5, 8, 9, 7, 4] + anumbers = [1, 4, 5, 8, 9, 7, 4] + len0seq: seq[int] = @[] + doAssert findIt(numbers, it == 4) == 1 + doAssert findIt(numbers, it > 9) == -1 + doAssert findIt(len0seq, true) == -1 + doAssert findIt(anumbers, it > 8) == 4 + doAssert findIt(anumbers, it > 9) == -1 + block: # anyIt let numbers = @[1, 4, 5, 8, 9, 7, 4] diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index db9fb80c44..dfa72faf22 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -789,12 +789,71 @@ bar block: # formatSize disableVm: when hasWorkingInt64: + doAssert formatSize(1024'i64 * 1024 * 1024 * 2 - 1) == "1.999GiB" + doAssert formatSize(1024'i64 * 1024 * 1024 * 2) == "2GiB" doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" # <=== bug #8231 - doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(int64.high) == "7.999EiB" + doAssert formatSize(int64.high div 2 + 1) == "4EiB" + doAssert formatSize(int64.high div 2) == "3.999EiB" + doAssert formatSize(int64.high div 4 + 1) == "2EiB" + doAssert formatSize(int64.high div 4) == "1.999EiB" + doAssert formatSize(int64.high div 8 + 1) == "1EiB" + doAssert formatSize(int64.high div 8) == "1023.999PiB" + doAssert formatSize(int64.high div 16 + 1) == "512PiB" + doAssert formatSize(int64.high div 16) == "511.999PiB" + doAssert formatSize(0) == "0B" + doAssert formatSize(0, includeSpace = true) == "0 B" + doAssert formatSize(1) == "1B" + doAssert formatSize(2) == "2B" + doAssert formatSize(1022) == "1022B" + doAssert formatSize(1023) == "1023B" + doAssert formatSize(1024) == "1KiB" + doAssert formatSize(1025) == "1.001KiB" + doAssert formatSize(1026) == "1.002KiB" + doAssert formatSize(1024 * 2 - 2) == "1.998KiB" + doAssert formatSize(1024 * 2 - 1) == "1.999KiB" + doAssert formatSize(1024 * 2) == "2KiB" + doAssert formatSize(1024 * 2 + 1) == "2.001KiB" + doAssert formatSize(1024 * 2 + 2) == "2.002KiB" + doAssert formatSize(4096 - 1) == "3.999KiB" doAssert formatSize(4096) == "4KiB" + doAssert formatSize(4096 + 1) == "4.001KiB" + doAssert formatSize(1024 * 512 - 1) == "511.999KiB" + doAssert formatSize(1024 * 512) == "512KiB" + doAssert formatSize(1024 * 512 + 1) == "512.001KiB" + doAssert formatSize(1024 * 1024 - 2) == "1023.998KiB" + doAssert formatSize(1024 * 1024 - 1) == "1023.999KiB" + doAssert formatSize(1024 * 1024) == "1MiB" + doAssert formatSize(1024 * 1024 + 1) == "1MiB" + doAssert formatSize(1024 * 1024 + 1023) == "1MiB" + doAssert formatSize(1024 * 1024 + 1024) == "1.001MiB" + doAssert formatSize(1024 * 1024 + 1024 * 2) == "1.002MiB" + doAssert formatSize(1024 * 1024 * 2 - 1) == "1.999MiB" + doAssert formatSize(1024 * 1024 * 2) == "2MiB" + doAssert formatSize(1024 * 1024 * 2 + 1) == "2MiB" + doAssert formatSize(1024 * 1024 * 2 + 1024) == "2.001MiB" + doAssert formatSize(1024 * 1024 * 2 + 1024 * 2) == "2.002MiB" + doAssert formatSize(1024 * 1024 * 4 - 1) == "3.999MiB" + doAssert formatSize(1024 * 1024 * 4) == "4MiB" + doAssert formatSize(1024 * (1024 * 4 + 1)) == "4.001MiB" + doAssert formatSize(1024 * 1024 * 512 - 1025) == "511.998MiB" + doAssert formatSize(1024 * 1024 * 512 - 1) == "511.999MiB" + doAssert formatSize(1024 * 1024 * 512) == "512MiB" + doAssert formatSize(1024 * 1024 * 512 + 1) == "512MiB" + doAssert formatSize(1024 * 1024 * 512 + 1024) == "512.001MiB" + doAssert formatSize(1024 * 1024 * 512 + 1024 * 2) == "512.002MiB" + doAssert formatSize(1024 * 1024 * 1024 - 1) == "1023.999MiB" + doAssert formatSize(1024 * 1024 * 1024) == "1GiB" + doAssert formatSize(1024 * 1024 * 1024 + 1) == "1GiB" + doAssert formatSize(1024 * 1024 * 1025) == "1.001GiB" + doAssert formatSize(1024 * 1024 * 1026) == "1.002GiB" + # != 2.234MiB as (2.234 * 1024 * 1024).int.float / (1024 * 1024) = 2.23399... + # and formatSize round down the value + doAssert formatSize((2.234*1024*1024).int) == "2.233MiB" doAssert formatSize(4096, prefix = bpColloquial, includeSpace = true) == "4 kB" doAssert formatSize(4096, includeSpace = true) == "4 KiB" - doAssert formatSize(5_378_934, prefix = bpColloquial, decimalSep = ',') == "5,13MB" + # (5378934).float / (1024 * 1024) = 5.12975... + doAssert formatSize(5_378_934, prefix = bpColloquial, decimalSep = ',') == "5,129MB" block: # formatEng disableVm: diff --git a/tests/stdlib/tsystem.nim b/tests/stdlib/tsystem.nim index ba05cb4286..3385d1107b 100644 --- a/tests/stdlib/tsystem.nim +++ b/tests/stdlib/tsystem.nim @@ -245,3 +245,32 @@ proc bar2() = static: bar2() bar2() + +when not defined(js): + proc foo = + block: + var s1:int = -10 + doAssertRaises(RangeDefect): + var n2:Natural = s1.Natural + + block: + var f = 751.0 + let m = f.int8 + + block: + var s2:float = -10 + doAssertRaises(RangeDefect): + var n2:Natural = s2.Natural + + + block: + type A = range[0..10] + + let f = 156.0 + + doAssertRaises(RangeDefect): + let a = f.A + + echo a # 156 + + foo() diff --git a/tools/debug/nim-gdb.py b/tools/debug/nim-gdb.py index 59e6ee99ce..2637481aee 100644 --- a/tools/debug/nim-gdb.py +++ b/tools/debug/nim-gdb.py @@ -4,6 +4,18 @@ import re import sys import traceback +# Add compatibility for older GDB versions +if not hasattr(gdb, 'SYMBOL_FUNCTION_DOMAIN'): + gdb.SYMBOL_FUNCTION_DOMAIN = 0 # This is the value used in newer GDB versions + +# Configure demangling for Itanium C++ ABI (which Nim uses) +try: + gdb.execute("set demangle-style gnu-v3") # GNU v3 style handles Itanium mangling + gdb.execute("set print asm-demangle on") + gdb.execute("set print demangle on") +except Exception as e: + gdb.write(f"Warning: Could not configure demangling: {str(e)}\n", gdb.STDERR) + # some feedback that the nim runtime support is loading, isn't a bad # thing at all. gdb.write("Loading Nim Runtime support.\n", gdb.STDERR) @@ -70,12 +82,12 @@ class NimTypeRecognizer: type_map_static = { 'NI': 'system.int', 'NI8': 'int8', 'NI16': 'int16', 'NI32': 'int32', 'NI64': 'int64', - + 'NU': 'uint', 'NU8': 'uint8','NU16': 'uint16', 'NU32': 'uint32', 'NU64': 'uint64', - + 'NF': 'float', 'NF32': 'float32', 'NF64': 'float64', - + 'NIM_BOOL': 'bool', 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string', 'NimStringV2': 'string' @@ -556,7 +568,7 @@ class NimSeqPrinter: except RuntimeError: inaccessible = True yield "data[{0}]".format(i), "inaccessible" - + ################################################################################ class NimArrayPrinter: