envPairs works in vm, nims (#18615)

* envPairs works in vm, nims

* fixup

compiler/vmops.nim

@@ -13,7 +13,7 @@ from std/math import sqrt, ln, log10, log2, exp, round, arccos, arcsin,
   arctan, arctan2, cos, cosh, hypot, sinh, sin, tan, tanh, pow, trunc,
   floor, ceil, `mod`, cbrt, arcsinh, arccosh, arctanh, erf, erfc, gamma,
   lgamma
-
+from std/sequtils import toSeq
 when declared(math.copySign):
   # pending bug #18762, avoid renaming math
   from std/math as math2 import copySign
@@ -22,8 +22,8 @@ when declared(math.signbit):
   # ditto
   from std/math as math3 import signbit
 
-from std/os import getEnv, existsEnv, delEnv, putEnv, dirExists, fileExists, walkDir,
-                   getAppFilename, raiseOSError, osLastError
+from std/os import getEnv, existsEnv, delEnv, putEnv, envPairs,
+  dirExists, fileExists, walkDir, getAppFilename, raiseOSError, osLastError
 
 from std/md5 import getMD5
 from std/times import cpuTime
@@ -156,6 +156,12 @@ proc stackTrace2(c: PCtx, msg: string, n: PNode) =
   stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info)
 
 proc registerAdditionalOps*(c: PCtx) =
+
+  template wrapIterator(fqname: string, iter: untyped) =
+    registerCallback c, fqname, proc(a: VmArgs) =
+      setResult(a, toLit(toSeq(iter)))
+
+
   proc gorgeExWrapper(a: VmArgs) =
     let ret = opGorge(getString(a, 0), getString(a, 1), getString(a, 2),
                          a.currentLineInfo, c.config)
@@ -341,3 +347,5 @@ proc registerAdditionalOps*(c: PCtx) =
     let p = a.getVar(0)
     let x = a.getFloat(1)
     addFloatSprintf(p.strVal, x)
+
+  wrapIterator("stdlib.os.envPairsImplSeq"): envPairs()

lib/pure/includes/osenv.nim

@@ -3,189 +3,200 @@
 when not declared(os) and not declared(ospaths):
   {.error: "This is an include file for os.nim!".}
 
-when defined(nodejs):
-  proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} =
-    var ret = default.cstring
-    let key2 = key.cstring
-    {.emit: "const value = process.env[`key2`];".}
-    {.emit: "if (value !== undefined) { `ret` = value };".}
-    result = $ret
+when not defined(nimscript):
+  when defined(nodejs):
+    proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} =
+      var ret = default.cstring
+      let key2 = key.cstring
+      {.emit: "const value = process.env[`key2`];".}
+      {.emit: "if (value !== undefined) { `ret` = value };".}
+      result = $ret
 
-  proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
-    var key2 = key.cstring
-    var ret: bool
-    {.emit: "`ret` = `key2` in process.env;".}
-    result = ret
+    proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
+      var key2 = key.cstring
+      var ret: bool
+      {.emit: "`ret` = `key2` in process.env;".}
+      result = ret
 
-  proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
-    var key2 = key.cstring
-    var val2 = val.cstring
-    {.emit: "process.env[`key2`] = `val2`;".}
+    proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
+      var key2 = key.cstring
+      var val2 = val.cstring
+      {.emit: "process.env[`key2`] = `val2`;".}
 
-  proc delEnv*(key: string) {.tags: [WriteEnvEffect].} =
-    var key2 = key.cstring
-    {.emit: "delete process.env[`key2`];".}
+    proc delEnv*(key: string) {.tags: [WriteEnvEffect].} =
+      var key2 = key.cstring
+      {.emit: "delete process.env[`key2`];".}
 
-  iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} =
-    var num: int
-    var keys: RootObj
-    {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".}
-    for i in 0..<num:
-      var key, value: cstring
-      {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".}
-      yield ($key, $value)
+    iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} =
+      var num: int
+      var keys: RootObj
+      {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".}
+      for i in 0..<num:
+        var key, value: cstring
+        {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".}
+        yield ($key, $value)
 
-# commented because it must keep working with js+VM
-# elif defined(js):
-#   {.error: "requires -d:nodejs".}
+  # commented because it must keep working with js+VM
+  # elif defined(js):
+  #   {.error: "requires -d:nodejs".}
 
-else:
-
-  proc c_getenv(env: cstring): cstring {.
-    importc: "getenv", header: "<stdlib.h>".}
-  when defined(windows):
-    proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "<stdlib.h>".}
-    from std/private/win_setenv import setEnvImpl
   else:
-    proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "<stdlib.h>".}
-  proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "<stdlib.h>".}
 
-  proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} =
-    ## Returns the value of the `environment variable`:idx: named `key`.
-    ##
-    ## If the variable does not exist, `""` is returned. To distinguish
-    ## whether a variable exists or it's value is just `""`, call
-    ## `existsEnv(key) proc <#existsEnv,string>`_.
-    ##
-    ## See also:
-    ## * `existsEnv proc <#existsEnv,string>`_
-    ## * `putEnv proc <#putEnv,string,string>`_
-    ## * `delEnv proc <#delEnv,string>`_
-    ## * `envPairs iterator <#envPairs.i>`_
-    runnableExamples:
-      assert getEnv("unknownEnv") == ""
-      assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist"
-
-    let env = c_getenv(key)
-    if env == nil: return default
-    result = $env
-
-  proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
-    ## Checks whether the environment variable named `key` exists.
-    ## Returns true if it exists, false otherwise.
-    ##
-    ## See also:
-    ## * `getEnv proc <#getEnv,string,string>`_
-    ## * `putEnv proc <#putEnv,string,string>`_
-    ## * `delEnv proc <#delEnv,string>`_
-    ## * `envPairs iterator <#envPairs.i>`_
-    runnableExamples:
-      assert not existsEnv("unknownEnv")
-
-    return c_getenv(key) != nil
-
-  proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
-    ## Sets the value of the `environment variable`:idx: named `key` to `val`.
-    ## If an error occurs, `OSError` is raised.
-    ##
-    ## See also:
-    ## * `getEnv proc <#getEnv,string,string>`_
-    ## * `existsEnv proc <#existsEnv,string>`_
-    ## * `delEnv proc <#delEnv,string>`_
-    ## * `envPairs iterator <#envPairs.i>`_
+    proc c_getenv(env: cstring): cstring {.
+      importc: "getenv", header: "<stdlib.h>".}
     when defined(windows):
-      if key.len == 0 or '=' in key:
-        raise newException(OSError, "invalid key, got: " & $(key, val))
-      if setEnvImpl(key, val, 1'i32) != 0'i32:
-        raiseOSError(osLastError(), $(key, val))
+      proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "<stdlib.h>".}
+      from std/private/win_setenv import setEnvImpl
     else:
-      if c_setenv(key, val, 1'i32) != 0'i32:
-        raiseOSError(osLastError(), $(key, val))
+      proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "<stdlib.h>".}
+    proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "<stdlib.h>".}
 
-  proc delEnv*(key: string) {.tags: [WriteEnvEffect].} =
-    ## Deletes the `environment variable`:idx: named `key`.
-    ## If an error occurs, `OSError` is raised.
-    ##
-    ## See also:ven
-    ## * `getEnv proc <#getEnv,string,string>`_
-    ## * `existsEnv proc <#existsEnv,string>`_
-    ## * `putEnv proc <#putEnv,string,string>`_
-    ## * `envPairs iterator <#envPairs.i>`_
-    template bail = raiseOSError(osLastError(), key)
-    when defined(windows):
-      #[ 
-      # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160
-      > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string
-      note that nil is not legal
-      ]#
-      if key.len == 0 or '=' in key:
-        raise newException(OSError, "invalid key, got: " & key)
-      if c_putenv_s(key, "") != 0'i32: bail
-    else:
-      if c_unsetenv(key) != 0'i32: bail
+    proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} =
+      ## Returns the value of the `environment variable`:idx: named `key`.
+      ##
+      ## If the variable does not exist, `""` is returned. To distinguish
+      ## whether a variable exists or it's value is just `""`, call
+      ## `existsEnv(key) proc <#existsEnv,string>`_.
+      ##
+      ## See also:
+      ## * `existsEnv proc <#existsEnv,string>`_
+      ## * `putEnv proc <#putEnv,string,string>`_
+      ## * `delEnv proc <#delEnv,string>`_
+      ## * `envPairs iterator <#envPairs.i>`_
+      runnableExamples:
+        assert getEnv("unknownEnv") == ""
+        assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist"
 
-  when defined(windows):
-    when useWinUnicode:
-      when defined(cpp):
-        proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)",
-            header: "<string.h>".}
+      let env = c_getenv(key)
+      if env == nil: return default
+      result = $env
+
+    proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
+      ## Checks whether the environment variable named `key` exists.
+      ## Returns true if it exists, false otherwise.
+      ##
+      ## See also:
+      ## * `getEnv proc <#getEnv,string,string>`_
+      ## * `putEnv proc <#putEnv,string,string>`_
+      ## * `delEnv proc <#delEnv,string>`_
+      ## * `envPairs iterator <#envPairs.i>`_
+      runnableExamples:
+        assert not existsEnv("unknownEnv")
+
+      return c_getenv(key) != nil
+
+    proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
+      ## Sets the value of the `environment variable`:idx: named `key` to `val`.
+      ## If an error occurs, `OSError` is raised.
+      ##
+      ## See also:
+      ## * `getEnv proc <#getEnv,string,string>`_
+      ## * `existsEnv proc <#existsEnv,string>`_
+      ## * `delEnv proc <#delEnv,string>`_
+      ## * `envPairs iterator <#envPairs.i>`_
+      when defined(windows):
+        if key.len == 0 or '=' in key:
+          raise newException(OSError, "invalid key, got: " & $(key, val))
+        if setEnvImpl(key, val, 1'i32) != 0'i32:
+          raiseOSError(osLastError(), $(key, val))
       else:
-        proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr",
-            header: "<string.h>".}
-    else:
-      proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr",
-          header: "<string.h>".}
-  elif defined(macosx) and not defined(ios) and not defined(emscripten):
-    # From the manual:
-    # Shared libraries and bundles don't have direct access to environ,
-    # which is only available to the loader ld(1) when a complete program
-    # is being linked.
-    # The environment routines can still be used, but if direct access to
-    # environ is needed, the _NSGetEnviron() routine, defined in
-    # <crt_externs.h>, can be used to retrieve the address of environ
-    # at runtime.
-    proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron",
-        header: "<crt_externs.h>".}
-  elif defined(haiku):
-    var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray
-  else:
-    var gEnv {.importc: "environ".}: cstringArray
+        if c_setenv(key, val, 1'i32) != 0'i32:
+          raiseOSError(osLastError(), $(key, val))
+
+    proc delEnv*(key: string) {.tags: [WriteEnvEffect].} =
+      ## Deletes the `environment variable`:idx: named `key`.
+      ## If an error occurs, `OSError` is raised.
+      ##
+      ## See also:ven
+      ## * `getEnv proc <#getEnv,string,string>`_
+      ## * `existsEnv proc <#existsEnv,string>`_
+      ## * `putEnv proc <#putEnv,string,string>`_
+      ## * `envPairs iterator <#envPairs.i>`_
+      template bail = raiseOSError(osLastError(), key)
+      when defined(windows):
+        #[ 
+        # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160
+        > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string
+        note that nil is not legal
+        ]#
+        if key.len == 0 or '=' in key:
+          raise newException(OSError, "invalid key, got: " & key)
+        if c_putenv_s(key, "") != 0'i32: bail
+      else:
+        if c_unsetenv(key) != 0'i32: bail
 
-  iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} =
-    ## Iterate over all `environments variables`:idx:.
-    ##
-    ## In the first component of the tuple is the name of the current variable stored,
-    ## in the second its value.
-    ##
-    ## See also:
-    ## * `getEnv proc <#getEnv,string,string>`_
-    ## * `existsEnv proc <#existsEnv,string>`_
-    ## * `putEnv proc <#putEnv,string,string>`_
-    ## * `delEnv proc <#delEnv,string>`_
     when defined(windows):
-      block:
-        template impl(get_fun, typ, size, zero, free_fun) =
-          let env = get_fun()
-          var e = env
-          if e == nil: break
-          while true:
-            let eend = strEnd(e)
-            let kv = $e
-            let p = find(kv, '=')
-            yield (substr(kv, 0, p-1), substr(kv, p+1))
-            e = cast[typ](cast[ByteAddress](eend)+size)
-            if typeof(zero)(eend[1]) == zero: break
-          discard free_fun(env)
-        when useWinUnicode:
-          impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW)
+      when useWinUnicode:
+        when defined(cpp):
+          proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)",
+              header: "<string.h>".}
         else:
-          impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA)
+          proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr",
+              header: "<string.h>".}
+      else:
+        proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr",
+            header: "<string.h>".}
+    elif defined(macosx) and not defined(ios) and not defined(emscripten):
+      # From the manual:
+      # Shared libraries and bundles don't have direct access to environ,
+      # which is only available to the loader ld(1) when a complete program
+      # is being linked.
+      # The environment routines can still be used, but if direct access to
+      # environ is needed, the _NSGetEnviron() routine, defined in
+      # <crt_externs.h>, can be used to retrieve the address of environ
+      # at runtime.
+      proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron",
+          header: "<crt_externs.h>".}
+    elif defined(haiku):
+      var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray
     else:
-      var i = 0
-      when defined(macosx) and not defined(ios) and not defined(emscripten):
-        var gEnv = NSGetEnviron()[]
-      while gEnv[i] != nil:
-        let kv = $gEnv[i]
-        inc(i)
-        let p = find(kv, '=')
-        yield (substr(kv, 0, p-1), substr(kv, p+1))
+      var gEnv {.importc: "environ".}: cstringArray
+
+    iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} =
+      when defined(windows):
+        block:
+          template impl(get_fun, typ, size, zero, free_fun) =
+            let env = get_fun()
+            var e = env
+            if e == nil: break
+            while true:
+              let eend = strEnd(e)
+              let kv = $e
+              let p = find(kv, '=')
+              yield (substr(kv, 0, p-1), substr(kv, p+1))
+              e = cast[typ](cast[ByteAddress](eend)+size)
+              if typeof(zero)(eend[1]) == zero: break
+            discard free_fun(env)
+          when useWinUnicode:
+            impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW)
+          else:
+            impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA)
+      else:
+        var i = 0
+        when defined(macosx) and not defined(ios) and not defined(emscripten):
+          var gEnv = NSGetEnviron()[]
+        while gEnv[i] != nil:
+          let kv = $gEnv[i]
+          inc(i)
+          let p = find(kv, '=')
+          yield (substr(kv, 0, p-1), substr(kv, p+1))
+
+proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops
+
+iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} =
+  ## Iterate over all `environments variables`:idx:.
+  ##
+  ## In the first component of the tuple is the name of the current variable stored,
+  ## in the second its value.
+  ##
+  ## Works in native backends, nodejs and vm, like the following APIs:
+  ## * `getEnv proc <#getEnv,string,string>`_
+  ## * `existsEnv proc <#existsEnv,string>`_
+  ## * `putEnv proc <#putEnv,string,string>`_
+  ## * `delEnv proc <#delEnv,string>`_
+  when nimvm:
+    for ai in envPairsImplSeq(): yield ai
+  else:
+    when defined(nimscript): discard
+    else:
+      for ai in envPairsImpl(): yield ai

lib/pure/os.nim

@@ -871,8 +871,7 @@ proc unixToNativePath*(path: string, drive=""): string {.
         inc(i)
 
 include "includes/oserr"
-when not defined(nimscript):
-  include "includes/osenv"
+include "includes/osenv"
 
 proc getHomeDir*(): string {.rtl, extern: "nos$1",
   tags: [ReadEnvEffect, ReadIOEffect].} =

testament/lib/stdtest/testutils.nim

@@ -1,5 +1,8 @@
 import std/private/miscdollars
-from std/os import getEnv
+when defined(nimscript):
+  import std/os # xxx investigate why needed
+else:
+  from std/os import getEnv
 import std/[macros, genasts]
 
 template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
@@ -26,7 +29,7 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
     msg2.add $expr & " " & msg
     echo msg2
 
-when not defined(js):
+when not defined(js) and not defined(nimscript):
   import std/strutils
 
   proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool =

tests/js/tos.nim

@@ -1,3 +1,5 @@
+# xxx consider merging this in tests/stdlib/tos.nim for increased coverage (with selecting disabling)
+
 static: doAssert defined(nodejs)
 
 import os
@@ -19,29 +21,3 @@ block:
   if not isWindows:
     doAssert cwd.isAbsolute
     doAssert relativePath(getCurrentDir() / "foo", "bar") == "../foo"
-
-import std/sequtils
-
-template main =
-  putEnv("foo", "bar")
-  doAssert getEnv("foo") == "bar"
-  doAssert existsEnv("foo")
-
-  putEnv("foo", "")
-  doAssert existsEnv("foo")
-  putEnv("foo", "bar2")
-  doAssert getEnv("foo") == "bar2"
-
-  when nimvm:
-    discard
-  else:
-    # need support in vmops: envPairs, delEnv
-    let s = toSeq(envPairs())
-    doAssert ("foo", "bar2") in s
-    doAssert ("foo", "bar") notin s
-
-    delEnv("foo")
-    doAssert not existsEnv("foo")
-
-static: main()
-main()

tests/stdlib/tosenv.nim

@@ -13,16 +13,18 @@ template main =
     for val in ["val", ""]: # ensures empty val works too
       const key = "NIM_TESTS_TOSENV_KEY"
       doAssert not existsEnv(key)
-      putEnv(key, val)
+
+      putEnv(key, "tempval")
+      doAssert existsEnv(key)
+      doAssert getEnv(key) == "tempval"
+
+      putEnv(key, val) # change a key that already exists
       doAssert existsEnv(key)
       doAssert getEnv(key) == val
-      when nimvm: discard
-      else:
-        doAssert (key, val) in toSeq(envPairs())
+
+      doAssert (key, val) in toSeq(envPairs())
       delEnv(key)
-      when nimvm: discard
-      else:
-        doAssert (key, val) notin toSeq(envPairs())
+      doAssert (key, val) notin toSeq(envPairs())
       doAssert not existsEnv(key)
       delEnv(key) # deleting an already deleted env var
       doAssert not existsEnv(key)
@@ -43,7 +45,7 @@ template main =
 static: main()
 main()
 
-when not defined(js):
+when not defined(js) and not defined(nimscript):
   block: # bug #18533
     proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".}
     var thr: Thread[void]

tests/test_nimscript.nims

@@ -71,7 +71,11 @@ import std/[
   decls, compilesettings, with, wrapnils
 ]
 
+# non-std imports
+import stdtest/testutils
+# tests (increase coverage via code reuse)
 import stdlib/trandom
+import stdlib/tosenv
 
 echo "Nimscript imports are successful."